Develop Reference

http proxy cache server not limiting browser functionality

I am trying to use this code to create an HTTP proxy cache server. When I run the code it starts running and connects to the port and everything but when I try to connect from the browser, for example, it opens a port on 55555 if I type in localhost:52523/www.google.com it works fine but when I try other sites specifically HTTP, for example, localhost:52523/www.microcenter.com or just localhost:52523/google.com it will display localhost didn’t send any data.
ERR_EMPTY_RESPONSE and shows an exception in the console though it creates the cache file on my computer.
I would like to find out how to edit the code so that I can access any website just as I would normally do on the browser without using the proxy server. It should be able to work with www.microcenter.com
import socket
import sys
import urllib
from urlparse import urlparse
Serv_Sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # socket.socket
function creates a socket.
port = Serv_Sock.getsockname()[1]
# Server socket created, bound and starting to listen
Serv_Sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # socket.socket
function creates a socket.
Serv_Sock.bind(('',port))
Serv_Sock.listen(5)
port = Serv_Sock.getsockname()[1]
# Prepare a server socket
print ("starting server on port %s...,"%(port))
def caching_object(splitMessage, Cli_Sock):
#this method is responsible for caching
Req_Type = splitMessage[0]
Req_path = splitMessage[1]
Req_path = Req_path[1:]
print "Request is ", Req_Type, " to URL : ", Req_path
#Searching available cache if file exists
url = urlparse(Req_path)
file_to_use = "/" + Req_path
print file_to_use
try:
file = open(file_to_use[5:], "r")
data = file.readlines()
print "File Present in Cache\n"
#Proxy Server Will Send A Response Message
#Cli_Sock.send("HTTP/1.0 200 OK\r\n")
#Cli_Sock.send("Content-Type:text/html")
#Cli_Sock.send("\r\n")
#Proxy Server Will Send Data
for i in range(0, len(data)):
print (data[i])
Cli_Sock.send(data[i])
print "Reading file from cache\n"
except IOError:
print "File Doesn't Exists In Cache\n fetching file from server \n
creating cache"
serv_proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host_name = Req_path
print "HOST NAME:", host_name
try:
serv_proxy.connect((url.host_name, 80))
print 'Socket connected to port 80 of the host'
fileobj = serv_proxy.makefile('r', 0)
fileobj.write("GET " + "http://" + Req_path + " HTTP/1.0\n\n")
# Read the response into buffer
buffer = fileobj.readlines()
# Create a new file in the cache for the requested file.
# Also send the response in the buffer to client socket
# and the corresponding file in the cache
tmpFile = open(file_to_use, "wb")
for data in buffer:
tmpFile.write(data)
tcpCliSock.send(data)
except:
print 'Illegal Request'
Cli_Sock.close()
while True:
# Start receiving data from the client
print 'Initiating server... \n Accepting connection\n'
Cli_Sock, addr = Serv_Sock.accept() # Accept a connection from client
#print addr
print ' connection received from: ', addr
message = Cli_Sock.recv(1024) #Recieves data from Socket
splitMessage = message.split()
if len(splitMessage) <= 1:
continue
caching_object(splitMessage, Cli_Sock)

There is a few errors in the code :-
The first is that a GET request does not expect the protocol to be passed in as part of the call, nor does it expect the host, instead the GET should be restricted to only the path + query string.
An additional HOST header should be added which specifies which host you are using (i.e www.google.com ) some web servers may be setup to ignore this and instead send you a default page, but results are intermittent.
You should have a peek at the HTTP RFC which gives some other headers that can be passed via HTTP.
You could also install something like Fiddler or Wireshark and monitor some sample HTTP calls and see how the payload is supposed to look.

Python closing sockets at random it seems

I've been looking and dealing with this issue for a week. I have client code that causes select() to return a socket that has actually closed from external reasons throwing an error 9 BAD FILE DESCRIPTOR, however I tested the code from a different python file and CANNOT get it to error. Ive tried a million things. heres a snippet from the server:
NOTE: This will work for a few iterations and then suddenly break, it errors out in the message_queue as key error due to the file descriptor breaking even tho a message/no message has a key for that socket present.
#Create the socket to communicate with uWSGI applications
server_address = ('localhost', 10001)
server = create_server_socket(server_address)
#Sockets which we expect to read on from select()
input_sockets = [server]
#Sockets which we expect to write to from select()
output_sockets = []
#Message buffer dicitonary for outgoing messages
message_queue = {}
#Now wait for connections endlessly
while input_sockets:
print >> sys.stderr, "Waiting for the next event..."
readable, writable, exceptional = select.select(input_sockets, output_sockets, input_sockets)
#Handle input_sockets
for s in readable:
#Server socket is available for reading now
if s is server:
#Create a connection and address object when incoming request is recieved
connection, client_addr = s.accept()
print >> sys.stderr, "Connection recieved from %s!" % (client_addr,)
#Set client connection to non blocking as well
connection.setblocking(0)
#Add this socket to input sockets as it will read for client data
input_sockets.append(connection)
#Give connection a queue for sending messages to it
message_queue[connection] = Queue.Queue()
#A client has sent data so we can handle its request
else:
#Pull data from the client
data = ""
try:
while True:
message = s.recv(1024)
if not message:
break
data += message
except Exception as e:
print str(e)
if data:
#Readable client socket has data
print >> sys.stderr, 'Recieved "%s" from %s' % (data, s.getpeername())
message_queue[s].put(data)
#Add output channel now to send message
if s not in output_sockets:
output_sockets.append(s)
#There is no data to be read, socket must be closed
else:
print >> sys.stderr, 'Closing', client_addr,'after recieving no data.'
#Stop listening for input on the socket
if s in output_sockets:
output_sockets.remove(s)
input_sockets.remove(s)
#Close the connection
s.close()
del message_queue[s]
#Handle writable connections
for s in writable:
if s:
try:
next_message = message_queue[s].get_nowait()
except:
print >> sys.stderr, 'No data to send for', s.getpeername()
output_sockets.remove(s)
else:
try:
print >> sys.stderr, 'Sending "%s" to %s' % (next_message, s.getpeername())
s.sendall(next_message)
except:
print >> sys.stderr, 'No data to send for', s.getpeername()
output_sockets.remove(s)
#s.sendall('EOF:!##$:EOF')
#Now handle any exceptions
for s in exceptional:
print >> sys.stderr, 'Handling exception on ', s.getpeername()
input_sockets.remove(s)
if s in output_sockets:
output_sockets.remove(s)
s.close()
#Remove any messages
del message_queue[s]
client:
messages = [ 'This is the message. ',
'It will be sent ',
'in parts.',
]
server_address = ('localhost', 10001)
# Create a TCP/IP socket
socks = [ socket.socket(socket.AF_INET, socket.SOCK_STREAM),
socket.socket(socket.AF_INET, socket.SOCK_STREAM),
]
# Connect the socket to the port where the server is listening
print >>sys.stderr, 'connecting to %s port %s' % server_address
for s in socks:
s.connect(server_address)
for message in messages:
# Send messages on both sockets
for s in socks:
print >>sys.stderr, '%s: sending "%s"' % (s.getsockname(), message)
s.send(message)
# Read responses on both sockets
for s in socks:
data = s.recv(1024)
print >>sys.stderr, '%s: received "%s"' % (s.getsockname(), data)
if not data:
print >>sys.stderr, 'closing socket', s.getsockname()
s.close()
NOTE: This client side is only to test and start passing messages.

There is a race in your code when a socket is returned as both readable and writable and you close the socket because the read returned 0 bytes. In this case you remove the socket from input_sockets, output_sockets and message_queue but the closed socket is still in writable and it will thus try to write to it inside the same iteration of the select loop.
I have no idea if this is the race you'll see because you neither show debug output not did you say where you stumble over this EBADF. To track similar problems down I recommend to augment your code with more debug information on where you close a socket and where you try to process a socket because it is readable or writable so that you actually find the exact place of the race when looking at the debug output.

Server do not response to multiple request from one client

I try to implement a server which only responds to one client at a time and client side. For the client side, I only used one connection and can send multiple requests to server. For the first request, everything goes well. For all the requests after the first one, the client will report error: socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
. I post my server and client code along with the sample test below:
The server part:
import socket
import re
#create server socket. AF_INET-> ipv4. SOCK_STREAM->socket type
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 20000
#bind the socket to localhost and a well-know port
serverSocket.bind((socket.gethostname(), port))
serverSocket.listen(3)
while True:
clientSocket, addr = serverSocket.accept()
print("Got a connection from %s" % str(addr))
request = clientSocket.recv(1024).decode()
print('Server received', repr(request))
splitRequest = re.split('\<|\>', request)
if splitRequest[0] == 'EXIT':
if len(splitRequest) == 1:
print "Normal exit"
elif len(splitRequest) > 1:
print splitRequest[1]
else:
if splitRequest[0] == 'GET':
fileName = splitRequest[1]
path = 'test_files/' + fileName
try :
with open(path, 'r') as serverFile:
message = serverFile.read()
print message
clientSocket.send(message)
except Exception as e:
message = str(e)
print message
clientSocket.send(message)
elif splitRequest[0] == 'BOUNCE':
message = splitRequest[1]
print message
clientSocket.sendall(message)
clientSocket.close()
The client side:
import socket
import re
def isValidRequest(input):
if re.match('GET\<.+\>', input) or re.match('BOUNCE\<.+\>', input) or input == 'EXIT' or re.match('EXIT\<.+\>', input):
return True
return False
def receiveAll(socket):
BUFF_SIZE = 4096 # 4 KiB
data = ""
while True:
part = socket.recv(BUFF_SIZE)
data += part
if len(part) < BUFF_SIZE:
# either 0 or end of data
break
return data
# create a client socket object
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# get local machine name
host = socket.gethostname()
# port number
port = 20000
# connection to hostname on the port.
s.connect((host, port))
while True:
request = raw_input()
if isValidRequest(request):
if request == 'EXIT' or re.match('EXIT\<.+\>', request):
s.send(request)
break
else:
s.send(request)
print "after send"
content = receiveAll(s)
print content
else:
print "Invalid request, please enter it again!"
# close client connection
s.close()
I run a test for request the same txt file-"MACD.txt" from server twice. The input in the console is "GET".The print message in the console for client:
*GET<MACD.txt>*
after send
MACD, short for moving average convergence/divergence, is a trading indicator used in technical analysis of stock prices, created by Gerald Appel in the late 1970s.[1] It is supposed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
*GET<MACD.txt>*
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
after send
The print message in the server part. And you can see server only print message for the first request:
Got a connection from ('192.168.126.1', 60567)
('Server received', "u'GET<MACD.txt>'")
MACD, short for moving average convergence/divergence, is a trading indicator used in technical analysis of stock prices, created by Gerald Appel in the late 1970s.[1] It is supposed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
I am confused by what I saw since I search the same problem in Stackoverflow and none of them match my case. I also read the document for python socket but still got nothing

Cache Proxy Server Returning 404 with www.google.com

I have a homework assignment which involves implementing a proxy cache server in Python for web pages. Here is my implementation of it
from socket import *
import sys
def main():
#Create a server socket, bind it to a port and start listening
tcpSerSock = socket(AF_INET, SOCK_STREAM) #Initializing socket
tcpSerSock.bind(("", 8030)) #Binding socket to port
tcpSerSock.listen(5) #Listening for page requests
while True:
#Start receiving data from the client
print 'Ready to serve...'
tcpCliSock, addr = tcpSerSock.accept()
print 'Received a connection from:', addr
message = tcpCliSock.recv(1024)
print message
#Extract the filename from the given message
filename = ""
try:
filename = message.split()[1].partition("/")[2].replace("/", "")
except:
continue
fileExist = False
try: #Check whether the file exists in the cache
f = open(filename, "r")
outputdata = f.readlines()
fileExist = True
#ProxyServer finds a cache hit and generates a response message
tcpCliSock.send("HTTP/1.0 200 OK\r\n")
tcpCliSock.send("Content-Type:text/html\r\n")
for data in outputdata:
tcpCliSock.send(data)
print 'Read from cache'
except IOError: #Error handling for file not found in cache
if fileExist == False:
c = socket(AF_INET, SOCK_STREAM) #Create a socket on the proxyserver
try:
srv = getaddrinfo(filename, 80)
c.connect((filename, 80)) #https://docs.python.org/2/library/socket.html
# Create a temporary file on this socket and ask port 80 for
# the file requested by the client
fileobj = c.makefile('r', 0)
fileobj.write("GET " + "http://" + filename + " HTTP/1.0\r\n")
# Read the response into buffer
buffr = fileobj.readlines()
# Create a new file in the cache for the requested file.
# Also send the response in the buffer to client socket and the
# corresponding file in the cache
tmpFile = open(filename,"wb")
for data in buffr:
tmpFile.write(data)
tcpCliSock.send(data)
except:
print "Illegal request"
else: #File not found
print "404: File Not Found"
tcpCliSock.close() #Close the client and the server sockets
main()
I configured my browsers to use my proxy server like so
But my problem when I run it is that no matter what web page I try to access it returns a 404 error with the initial connection and then a connection reset error with subsequent connections. I have no idea why so any help would be greatly appreciated, thanks!

There are quite a number of issues with your code.
Your URL parser is quite cumbersome. Instead of the line
filename = message.split()[1].partition("/")[2].replace("/", "")
I would use
import re
parsed_url = re.match(r'GET\s+http://(([^/]+)(.*))\sHTTP/1.*$', message)
local_path = parsed_url.group(3)
host_name = parsed_url.group(2)
filename = parsed_url.group(1)
If you catch an exception there, you should probably throw an error because it is a request your proxy doesn't understand (e.g. a POST).
When you assemble your request to the destination server, you then use
fileobj.write("GET {object} HTTP/1.0\n".format(object=local_path))
fileobj.write("Host: {host}\n\n".format(host=host_name))
You should also include some of the header lines from the original request because they can make a major difference to the returned content.
Furthermore, you currently cache the entire response with all header lines, so you should not add your own when serving from cache.
What you have doesn't work, anyway, because there is no guarantee that you will get a 200 and text/html content. You should check the response code and only cache if you did indeed get a 200.

HTTP Proxy server [Python]

I have this code here for an HTTP Proxy server which works. How do I create another program called "Client"? The client will send HTTP GET requests to multiple web servers
via the proxy server. The client program connects to the proxy and sends HTTP GET requests for the following 3 websites: (www.google.com, www.yahoo.com, www.stackoverflow.com)
with an interval of 30 seconds.
-My overall question is how do i send HTTP GET requests to the proxy server from python, not my web browser?
OSX 10.10.3 Python 3.4
When i call this proxy in my terminal:
python 1869.py 2000
You can give any port number in place of 2000.
Output:
starting server ....
Initiating server...
Accepting connection
Then in my browser (im using the most update version of chrome) I type in:
localhost:2000/www.stackoverflow.com
And my terminal output is:
request is GET to URL : www.stackoverflow.com
/www.stackoverflow.com
File Present in Cache
HTTP/1.1 301 Moved Permanently
Content-Type: text/html; charset=UTF-8
Location: http://stackoverflow.com/
Date: Thu, 07 May 2015 17:45:40 GMT
Content-Length: 148
Connection: close
Age: 0
<head><title>Document Moved</title></head>
<body><h1>Object Moved</h1>This document may be found here</body>
Reading file from cache
Initiating server...
Accepting connection
Proxy code:
import socket
import sys
if len(sys.argv) <= 1:
print 'Usage: "python S.py port"\n[port : It is the port of the Proxy Server'
sys.exit(2)
# Server socket created, bound and starting to listen
Serv_Port = int(sys.argv[1]) # sys.argv[1] is the port number entered by the user
Serv_Sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # socket.socket function creates a socket.
# Prepare a server socket
print "starting server ...."
Serv_Sock.bind(('', Serv_Port))
Serv_Sock.listen(5)
def caching_object(splitMessage, Cli_Sock):
#this method is responsible for caching
Req_Type = splitMessage[0]
Req_path = splitMessage[1]
Req_path = Req_path[1:]
print "Request is ", Req_Type, " to URL : ", Req_path
#Searching available cache if file exists
file_to_use = "/" + Req_path
print file_to_use
try:
file = open(file_to_use[1:], "r")
data = file.readlines()
print "File Present in Cache\n"
#Proxy Server Will Send A Response Message
#Cli_Sock.send("HTTP/1.0 200 OK\r\n")
#Cli_Sock.send("Content-Type:text/html")
#Cli_Sock.send("\r\n")
#Proxy Server Will Send Data
for i in range(0, len(data)):
print (data[i])
Cli_Sock.send(data[i])
print "Reading file from cache\n"
except IOError:
print "File Doesn't Exists In Cache\n fetching file from server \n creating cache"
serv_proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host_name = Req_path
print "HOST NAME:", host_name
try:
serv_proxy.connect((host_name, 80))
print 'Socket connected to port 80 of the host'
fileobj = serv_proxy.makefile('r', 0)
fileobj.write("GET " + "http://" + Req_path + " HTTP/1.0\n\n")
# Read the response into buffer
buffer = fileobj.readlines()
# Create a new file in the cache for the requested file.
# Also send the response in the buffer to client socket
# and the corresponding file in the cache
tmpFile = open("./" + Req_path, "wb")
for i in range(0, len(buffer)):
tmpFile.write(buffer[i])
Cli_Sock.send(buffer[i])
except:
print 'Illegal Request'
Cli_Sock.close()
while True:
# Start receiving data from the client
print 'Initiating server... \n Accepting connection\n'
Cli_Sock, addr = Serv_Sock.accept() # Accept a connection from client
#print addr
print ' connection received from: ', addr
message = Cli_Sock.recv(1024) #Recieves data from Socket
splitMessage = message.split()
if len(splitMessage) <= 1:
continue
caching_object(splitMessage, Cli_Sock)

you can use httpie from linux' terminal:
http [get/post] http://host:port/link_name/
in your app you can use requests:
pip install requests
import requests
response = requests.get(url='url', proxies='proxies')
response.close()
print(response)