Develop Reference

Write proxy web server using socket in python

This is my code for my seminar at university in computer network course.
Require: write a proxy web server. Receive HTTP request from the browser port 8888 and send HTTP request to web server port 80. It seems like I had trouble sending request to web server.
Could you show me my error in this situation?
Many thanks
import socket
import sys
import _thread
import traceback
import ssl
def CreateServer(host, port):
Server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
Server.bind((host,port))
Server.listen(5)
return Server
def ReadRequest(Client):
re = ""
Client.settimeout(10.0)
try:
re = Client.recv(1024).decode()
while (re):
re = re + Client.recv(1024).decode()
except socket.timeout: # fail after 1 second of no activity
if not re:
print("Didn't receive data! [Timeout]")
finally:
return re
#2. Client connect Server + 3. Read HTTP Request
def ReadHTTPRequest(Server):
re = ""
while (re == ""):
Client, address = Server.accept()
print("Client: ", address," da ket noi toi Server")
re = ReadRequest(Client)
return Client,address, re
def proxy_server(webserver, port, conn, data, addr):
print("{} {} {} {}".format(webserver, port, conn, addr))
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(10.0)
s.connect((webserver, port))
s.sendall(data)
while 1:
reply = s.recv(1024)
if len(reply) > 0:
conn.send(reply)
print("[*] Request sent: {} > {}".format(addr[0],webserver))
else:
break
s.close()
conn.close()
except Exception as e:
print(e)
traceback.print_exc()
s.close()
conn.close()
sys.exit(1)
def conn_string(Client,Request,addr):
try:
#print(addr)
first_line=Request.split('\n')[0]
url=first_line.split(" ")[1]
http_pos=url.find("://")
if http_pos==-1:
temp=url
else:
temp=url[(http_pos+3):]
port_pos=temp.find(":")
webserver_pos=temp.find("/")
if webserver_pos == -1:
webserver_pos = len(temp)
webserver = ""
port = -1
if port_pos == -1 or webserver_pos < port_pos:
port = 80
webserver = temp[:webserver_pos]
else:
port = 80
#port = int(temp[(port_pos + 1):][:webserver_pos - port_pos -1])
webserver = temp[:port_pos]
proxy_server(webserver,port,Client,Request.encode(),addr)
except Exception as e:
print (e)
traceback.print_exc()
if __name__=="__main__":
try:
Server=CreateServer("",8888)
print("[*] Intializing socket. Done.")
print("[*] Socket binded successfully...")
print("[*] Server started successfully [{}]".format(8888))
except Exception as e:
print(e)
sys.exit(2)
while True:
try:
Client,addr,Request=ReadHTTPRequest(Server)
print("---------HTTP request: ")
print(Request)
_thread.start_new_thread(conn_string,(Client,Request,addr))
except KeyboardInterrupt:
Server.close()
print("\n[*] Shutting down..")
sys.exit()
Server.close()
[1]: https://i.stack.imgur.com/216ZO.png

You are simply forwarding the original request to the server, i.e. with the full URL inside the request:
GET http://ktdbcl.hcmus.edu.vn/ HTTP/1.1
Host: ktdbcl.hcmus.edu.vn
...
Instead only the path should be forwarded:
GET / HTTP/1.1
Host: ktdbcl.hcmus.edu.vn
...
Apart from that the reading of request and response is severely broken. You don't properly parse the HTTP protocol but instead simply wait some time and treat no data for some seconds as end of message. Since today multiple requests and responses will be done over the same connection your code will thus severely slow down any browsing.
Instead of waiting for multiple seconds for no data as the marker for end of request/response you should properly parse the HTTP protocol and wait for the appropriate end of request marker defined by the HTTP standard - see RFC 7230.

Python3 NAT hole punching

I know this topic is not new. There is various information out there although, the robust solution is not presented (at least I did not found). I have a P2P daemon written in python3 and the last element on the pie is to connect two clients behind the NAT via TCP. My references for this topic:
https://bford.info/pub/net/p2pnat/
How to make 2 clients connect each other directly, after having both connected a meeting-point server?
Problems with TCP hole punching
What I have done so far:
SERVER:
#!/usr/bin/env python3
import threading
import socket
MY_AS_SERVER_PORT = 9001
TIMEOUT = 120.0
BUFFER_SIZE = 4096
def get_my_local_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return bytes(IP, encoding='utf-8')
def wait_for_msg(new_connection, client_address):
while True:
try:
packet = new_connection.recv(BUFFER_SIZE)
if packet:
msg_from_client = packet.decode('utf-8')
client_connected_from_ip = client_address[0]
client_connected_from_port = client_address[1]
print("We have a client. Client advertised his local IP as:", msg_from_client)
print(f"Although, our connection is from: [{client_connected_from_ip}]:{client_connected_from_port}")
msg_back = bytes("SERVER registered your data. Your local IP is: " + str(msg_from_client) + " You are connecting to the server FROM: " + str(client_connected_from_ip) + ":" + str(client_connected_from_port), encoding='utf-8')
new_connection.sendall(msg_back)
break
except ConnectionResetError:
break
except OSError:
break
def server():
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock.bind((get_my_local_ip().decode('utf-8'), MY_AS_SERVER_PORT))
sock.listen(8)
sock.settimeout(TIMEOUT)
while True:
try:
new_connection, client_address = sock.accept()
if new_connection:
threading.Thread(target=wait_for_msg, args=(new_connection,client_address,)).start()
# print("connected!")
# print("")
# print(new_connection)
# print("")
# print(client_address)
msg = bytes("Greetings! This message came from SERVER as message back!", encoding='utf-8')
new_connection.sendall(msg)
except socket.timeout:
pass
if __name__ == '__main__':
server()
CLIENT:
#!/usr/bin/python3
import sys
import socket
import time
import threading
SERVER_IP = '1.2.3.4'
SERVER_PORT = 9001
# We don't want to establish a connection with a static port. Let the OS pick a random empty one.
#MY_AS_CLIENT_PORT = 8510
TIMEOUT = 3
BUFFER_SIZE = 4096
def get_my_local_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return bytes(IP, encoding='utf-8')
def constantly_try_to_connect(sock):
while True:
try:
sock.connect((SERVER_IP, SERVER_PORT))
except ConnectionRefusedError:
print(f"Can't connect to the SERVER IP [{SERVER_IP}]:{SERVER_PORT} - does the server alive? Sleeping for a while...")
time.sleep(1)
except OSError:
#print("Already connected to the server. Kill current session to reconnect...")
pass
def client():
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
#sock.bind((get_my_local_ip().decode('utf-8'), MY_AS_CLIENT_PORT))
sock.settimeout(TIMEOUT)
threading.Thread(target=constantly_try_to_connect, args=(sock,)).start()
while True:
try:
packet = sock.recv(BUFFER_SIZE)
if packet:
print(packet)
sock.sendall(get_my_local_ip())
except OSError:
pass
if __name__ == '__main__':
client()
Now the current code results:
./tcphole_server.py
We have a client. Client advertised his local IP as: 10.10.10.50
Although, our connection is from: [89.22.11.50]:32928
We have a client. Client advertised his local IP as: 192.168.1.20
Although, our connection is from: [78.88.77.66]:51928
./tcphole_client1.py
b'Greetings! This message came from SERVER as message back!'
b'SERVER registered your data. Your local IP is: 192.168.1.20 You are connecting to the server FROM: 89.22.11.50:32928'
./tcphole_client2.py
b'Greetings! This message came from SERVER as message back!'
b'SERVER registered your data. Your local IP is: 10.10.10.50 You are connecting to the server FROM: 78.88.77.66:51928'
As you can see the server has all information to connect two clients. We can send details about the other peer individually through the current server-client connection.
Now two questions remain in my head:
Assuming the SERVER sends information about CLIENT 1 and CLIENT 2 for each of the peers. And now the CLIENTS starts connecting like [89.22.11.50]:32928 <> [78.88.77.66]:51928 Does the SERVER should close the current connections with the CLIENTS?
How the CLIENT Router behaves? I assume it expecting the same EXTERNAL SERVER SRC IP [1.2.3.4], instead gets one of the CLIENTS EXT IP for instance [89.22.11.50] or [78.88.77.66]?
This is messier than I thought. Any help to move forward appreciated. Hope this would help other Devs/DevOps too.

Finally found the expected behavior! Don't want to give too much code here but I hope after this you will understand the basics of how to implement it. Best to have a separate file in each of the client's folder - nearby ./tcphole_client1.py and ./tcphole_client2.py. We need to connect fast after we initiated sessions with the SERVER. Now for instance:
./tcphole_client_connector1.py 32928 51928
./tcphole_client_connector2.py 51928 32928
Remember? We need to connect to the same ports as we initiated with SERVER:
[89.22.11.50]:32928 <> [78.88.77.66]:51928
The first port is needed to bind the socket (OUR). With the second port, we are trying to connect to the CLIENT. The other CLIENT doing the same procedure except it binds to his port and connects to yours bound port. If the ROUTER still has an active connection - SUCCESS.

Server do not response to multiple request from one client

I try to implement a server which only responds to one client at a time and client side. For the client side, I only used one connection and can send multiple requests to server. For the first request, everything goes well. For all the requests after the first one, the client will report error: socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
. I post my server and client code along with the sample test below:
The server part:
import socket
import re
#create server socket. AF_INET-> ipv4. SOCK_STREAM->socket type
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 20000
#bind the socket to localhost and a well-know port
serverSocket.bind((socket.gethostname(), port))
serverSocket.listen(3)
while True:
clientSocket, addr = serverSocket.accept()
print("Got a connection from %s" % str(addr))
request = clientSocket.recv(1024).decode()
print('Server received', repr(request))
splitRequest = re.split('\<|\>', request)
if splitRequest[0] == 'EXIT':
if len(splitRequest) == 1:
print "Normal exit"
elif len(splitRequest) > 1:
print splitRequest[1]
else:
if splitRequest[0] == 'GET':
fileName = splitRequest[1]
path = 'test_files/' + fileName
try :
with open(path, 'r') as serverFile:
message = serverFile.read()
print message
clientSocket.send(message)
except Exception as e:
message = str(e)
print message
clientSocket.send(message)
elif splitRequest[0] == 'BOUNCE':
message = splitRequest[1]
print message
clientSocket.sendall(message)
clientSocket.close()
The client side:
import socket
import re
def isValidRequest(input):
if re.match('GET\<.+\>', input) or re.match('BOUNCE\<.+\>', input) or input == 'EXIT' or re.match('EXIT\<.+\>', input):
return True
return False
def receiveAll(socket):
BUFF_SIZE = 4096 # 4 KiB
data = ""
while True:
part = socket.recv(BUFF_SIZE)
data += part
if len(part) < BUFF_SIZE:
# either 0 or end of data
break
return data
# create a client socket object
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# get local machine name
host = socket.gethostname()
# port number
port = 20000
# connection to hostname on the port.
s.connect((host, port))
while True:
request = raw_input()
if isValidRequest(request):
if request == 'EXIT' or re.match('EXIT\<.+\>', request):
s.send(request)
break
else:
s.send(request)
print "after send"
content = receiveAll(s)
print content
else:
print "Invalid request, please enter it again!"
# close client connection
s.close()
I run a test for request the same txt file-"MACD.txt" from server twice. The input in the console is "GET".The print message in the console for client:
*GET<MACD.txt>*
after send
MACD, short for moving average convergence/divergence, is a trading indicator used in technical analysis of stock prices, created by Gerald Appel in the late 1970s.[1] It is supposed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
*GET<MACD.txt>*
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
after send
The print message in the server part. And you can see server only print message for the first request:
Got a connection from ('192.168.126.1', 60567)
('Server received', "u'GET<MACD.txt>'")
MACD, short for moving average convergence/divergence, is a trading indicator used in technical analysis of stock prices, created by Gerald Appel in the late 1970s.[1] It is supposed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
I am confused by what I saw since I search the same problem in Stackoverflow and none of them match my case. I also read the document for python socket but still got nothing

Python Server send data not working

I am currently working on a server in Python, the problem I am facing is the client could not retrieve the sent data from server.
The code of the server is:
import sys
import socket
from threading import Thread
allClients=[]
class Client(Thread):
def __init__(self,clientSocket):
Thread.__init__(self)
self.sockfd = clientSocket #socket client
self.name = ""
self.nickName = ""
def newClientConnect(self):
allClients.append(self.sockfd)
while True:
while True:
try:
rm= self.sockfd.recv(1024)
print rm
try:
self.sockfd.sendall("\n Test text to check send.")
print "Data send successfull"
break
except socket.error, e:
print "Could not send data"
break
except ValueError:
self.sockfd.send("\n Could not connect properly")
def run(self):
self.newClientConnect()
self.sockfd.close()
while True:
buff = self.sockfd.recv(1024)
if buff.strip() == 'quit':
self.sockfd.close()
break # Exit when break
else:
self.sendAll(buff)
#Main
if __name__ == "__main__":
#Server Connection to socket:
IP = '127.0.0.1'
PORT = 80
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
print ("Server Started")
try:
serversocket.bind(('',5000))
except ValueError,e:
print e
serversocket.listen(5)
while True:
(clientSocket, address) = serversocket.accept()
print 'New connection from ', address
ct = Client(clientSocket)
ct.start()
__all__ = ['allClients','Client']
#--
And the client connecting is:
import socket
HOST = '192.168.1.4' # The remote host
PORT = 5000 # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
data = s.recv(1024)
s.close()
print 'Received', data#repr(data)
In need of a quick solution....
Thanks,

I tested out your code, and when I commented out
rm= self.sockfd.recv(1024)
print rm
it worked fine. Basically the server stopped there to wait for a message that never came. If it still does not work for you, there might be two problems. Either you have a firewall that blocks the connection somehow, or you have old servers running in the background from previous tries that actually wasn't killed. Check your processes if pythonw.exe or equivalent is running when it shouldn't be, and kill it.

To wait for response:
with s.makefile('rb') as f:
data = f.read() # block until the whole response is read
s.close()
There are multiple issues in your code:
nested while True without break
finally: ..close() is executed before except ValueError: ..send
multiple self.sockfd.close()
etc
Also you should probably use .sendall() instead of .send().

your server code is excepting client send something first,
rm= self.sockfd.recv(1024)
but I don't see any in your code
please try send something in your client code
s.connect((HOST, PORT))
s.send("hello")

Short solution
Add a short sleep after connect.
import time
time.sleep(3)

How to keep a socket open until client closes it?

I have simple python server and client.
Server:
import SocketServer
import threading
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "localhost", 3288
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
server.serve_forever()
Client:
import socket
import sys
from time import sleep
HOST, PORT = "localhost", 3288
data = "hello"
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
sock.send(data + "\n")
received = sock.recv(1024)
sleep(10)
sock.send(data + "\n")
received = sock.recv(1024)
sleep(10)
sock.send(data + "\n")
received = sock.recv(1024)
finally:
sock.close()
Here is the output I get:
Server:
>python server.py
127.0.0.1 wrote:
hello
Client:
>python client.py
Traceback (most recent call last):
File "client.py", line 18, in <module>
received = sock.recv(1024)
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
I tried it on a linux machine as well. The server only receives one message and then I get an error on the recv statement of second message. I have just started learning networking on python but I think the server is closing the socket for some reason. How do I correct this?

A MyTcpHandler object is created for each connection, and handle is called to deal with the client. The connection is closed when handle returns, so you have to handle the complete communication from the client within the handle method:
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
while 1:
self.data = self.request.recv(1024)
if not self.data:
break
self.data = self.data.strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
NOTE: recv returns '' when the client closes the connection, so I moved .strip() after the recv so there is no false alarm due to the client sending only white space.

I'll first admit that it's been years since I last used SocketServer, so there might be more idiomatic approaches to solve your problem.
Note that your client opens a single connection and sends three sets of data and receives three sets of data. (Hopefully the TCP stack will send buffered data once you call receive() on the socket.)
Your server is expecting to handle a client connection completely, from start to finish, when it is called from the SocketServer callback mechanism. Your current class does a little bit of IO and then quits. You just need to extend your server callback to do more:
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
foo = self.request.recv(1024).strip()
self.request.send(foo.lower())
bar = self.request.recv(1024).strip()
self.request.send("goodbye " + bar)

TO a similar problem here error: [Errno 10053]
I also tried the same thing and got the same error.
If there is a simple code like this to demonstrate this error:
import socket
host = 'localhost'
port = 5001
size = 102400
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
for msg in ['Hello, world','Test','anything goes here']:
s.send(msg)
data = s.recv(size)
print 'Received:', data
s.close()
If you create a socket object and the amt it can send and echo back from server to see how much it receivers, if you vary that, say 1024 to 102400(in this code);
Which means the socket should not get closed but again in my Windows OS, the server side keeps listening and printing any data that client sends but on the Client side you get this error;
However if it is that the client can connect only once and send and receive only once, then that is how it was designed. Trying this works without any errors:
for msg in ['Hello, world','Test','anything goes here']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
s.send(msg)
data = s.recv(size)
s.close()
print 'Received:', data
I am not sure if one socket object works only once to send and recieve data.
UPDATE
I think the issue was the capacity per client socket to receive data as per the buffersize fixed;
That's why the second code snippet above works thus creating new client connection sockets on the server. But that way lots of sockets are going to get used up.
Instead the following code fixed that problem by checking the amt of size being used up. If it exceeds the given amount, it creates a new socket at clients' but makes sure the message is sent; Actually the problem was with the server code but fixed it.
size = 10
This is a quick baby attempt at the code. I am sure you would understand and optimize it for the better!
client code:
messag = ['Hello, world', 'Test', 'anything goes here']
def client_to_server(messag,host,port,size):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
countmsg = 0
restmsg = ''
for msg in messag:
strl = tmsg = msg
if len(restmsg):
tmsg = restmsg + ' ' + msg
countmsg = len(tmsg)
if countmsg <= size:
pass
else:
restmsg = tmsg[size:]
tmsg = tmsg[:size]
#s.close()
countmsg = len(tmsg)
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((host, port))
print 'Sending to server msg {}'.format(tmsg)
s.send(tmsg)
# s.settimeout(1)
try:
data = s.recv(size)
print 'Received:', data
if strl == data:
print strl,data
countmsg = 0
restmsg = ''
except (socket.error), e:
print e.args,e.message
s.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.close()
if restmsg:
client_to_server([restmsg],host,port,size)
return
client_to_server(messag,host,port,size)
Server Code:
size = 1024 #This has to be bigger than client buf size!
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(backlog)
while True:
#this is what accepts and creates a P2P dedicated client socket per socket
client, address = s.accept()
try:
data = client.recv(size)
while data or 0:
print "Client sent {} | Server sending data to client address {}".format(data, address)
client.send(data)
data = client.recv(size)
else: client.close()
except (socket.error), e:
client.close()
print e.args, e.message
Try it out. This uses the same socket.