Develop Reference

How to visualize the body part of a BaseHTTPRequestHandler

I'm writing a server in Python and I want to send some data in the body part of a GET request. Currently when I run my code it just freezes.
I tried to put it in a try/except and read details about the instance variable rfile but I didn't find anything helpful
Client-side :
import http.client
import sys
import os
#get http server ip
http_server = sys.argv[1]
#create a connection
conn = http.client.HTTPConnection(http_server)
while 1:
cmd = input('input command (ex. GET index.html): ')
cmd = cmd.split()
f = open('data.txt')
if cmd[0] == 'exit': #tipe exit to end it
break
#request command to server
conn.request(cmd[0],'',f.read())
#get response from server
rsp = conn.getresponse()
#print server response and data
print(rsp.status, rsp.reason)
data_received = rsp.read()
print(data_received)
Server-side :
from http.server import BaseHTTPRequestHandler,HTTPServer
import os
class TestHTTPRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
a = ''
fichier = open("data2.txt", "w")
try:
fichier.write(self.rfile.read())
except:
self.send_response(200)
self.send_header('Content-type','text-html')
self.end_headers()
return
def run():
print('http server is starting...')
server_address = ('127.0.0.1',80)
httpd = HTTPServer(server_address, PancakeHTTPRequestHandler)
print('htttp server is running...')
httpd.serve_forever()
if __name__ == '__main__':
run()
I expect being able to write my data from my GET request in my data2.txt file.
Thank you for your help

It freezes because of the self.rfile.read() in your server.py. The read method expect either EOF or a byte length to read. See https://docs.python.org/3/library/io.html#io.BufferedIOBase.read
You're trying to log the client requests made to the server, a quick work around would be to pass the content length of the request to the read method with int(self.headers.get('Content-Length'))
In the end it gives us:
client.py
import http.client
import sys
import os
#get http server ip
http_server = sys.argv[1]
#create a connection
conn = http.client.HTTPConnection(http_server)
while 1:
cmd = input('input command (ex. GET index.html): ')
cmd = cmd.split()
f = open('data.txt')
if cmd[0] == 'exit': #tipe exit to end it
break
#request command to server
conn.request(cmd[0], '', f.read())
#get response from server
rsp = conn.getresponse()
#print server response and data
print(rsp.status, rsp.reason)
data_received = rsp.read()
print(data_received)
server.py
from http.server import BaseHTTPRequestHandler,HTTPServer
import os
class TestHTTPRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
a = ''
fichier = open("data2.txt", "a")
try:
content_length = int(self.headers.get('Content-Length'))
response_str = self.rfile.read(content_length)
fichier.write(response_str.decode('utf-8'))
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
except:
self.send_response(200)
self.send_header('Content-type','text-html')
self.end_headers()
fichier.close()
return
def run():
print('http server is starting...')
server_address = ('127.0.0.1',80)
httpd = HTTPServer(server_address, TestHTTPRequestHandler)
print('htttp server is running...')
httpd.serve_forever()
if __name__ == '__main__':
run()
ps. I don't know what PancakeHTTPRequestHandler was so I replaced it with TestHTTPRequestHandler. And I also added a response in try except on the server side so that the the client gets a response otherwise it will crash.

Cache Proxy Server Returning 404 with www.google.com

I have a homework assignment which involves implementing a proxy cache server in Python for web pages. Here is my implementation of it
from socket import *
import sys
def main():
#Create a server socket, bind it to a port and start listening
tcpSerSock = socket(AF_INET, SOCK_STREAM) #Initializing socket
tcpSerSock.bind(("", 8030)) #Binding socket to port
tcpSerSock.listen(5) #Listening for page requests
while True:
#Start receiving data from the client
print 'Ready to serve...'
tcpCliSock, addr = tcpSerSock.accept()
print 'Received a connection from:', addr
message = tcpCliSock.recv(1024)
print message
#Extract the filename from the given message
filename = ""
try:
filename = message.split()[1].partition("/")[2].replace("/", "")
except:
continue
fileExist = False
try: #Check whether the file exists in the cache
f = open(filename, "r")
outputdata = f.readlines()
fileExist = True
#ProxyServer finds a cache hit and generates a response message
tcpCliSock.send("HTTP/1.0 200 OK\r\n")
tcpCliSock.send("Content-Type:text/html\r\n")
for data in outputdata:
tcpCliSock.send(data)
print 'Read from cache'
except IOError: #Error handling for file not found in cache
if fileExist == False:
c = socket(AF_INET, SOCK_STREAM) #Create a socket on the proxyserver
try:
srv = getaddrinfo(filename, 80)
c.connect((filename, 80)) #https://docs.python.org/2/library/socket.html
# Create a temporary file on this socket and ask port 80 for
# the file requested by the client
fileobj = c.makefile('r', 0)
fileobj.write("GET " + "http://" + filename + " HTTP/1.0\r\n")
# Read the response into buffer
buffr = fileobj.readlines()
# Create a new file in the cache for the requested file.
# Also send the response in the buffer to client socket and the
# corresponding file in the cache
tmpFile = open(filename,"wb")
for data in buffr:
tmpFile.write(data)
tcpCliSock.send(data)
except:
print "Illegal request"
else: #File not found
print "404: File Not Found"
tcpCliSock.close() #Close the client and the server sockets
main()
I configured my browsers to use my proxy server like so
But my problem when I run it is that no matter what web page I try to access it returns a 404 error with the initial connection and then a connection reset error with subsequent connections. I have no idea why so any help would be greatly appreciated, thanks!

There are quite a number of issues with your code.
Your URL parser is quite cumbersome. Instead of the line
filename = message.split()[1].partition("/")[2].replace("/", "")
I would use
import re
parsed_url = re.match(r'GET\s+http://(([^/]+)(.*))\sHTTP/1.*$', message)
local_path = parsed_url.group(3)
host_name = parsed_url.group(2)
filename = parsed_url.group(1)
If you catch an exception there, you should probably throw an error because it is a request your proxy doesn't understand (e.g. a POST).
When you assemble your request to the destination server, you then use
fileobj.write("GET {object} HTTP/1.0\n".format(object=local_path))
fileobj.write("Host: {host}\n\n".format(host=host_name))
You should also include some of the header lines from the original request because they can make a major difference to the returned content.
Furthermore, you currently cache the entire response with all header lines, so you should not add your own when serving from cache.
What you have doesn't work, anyway, because there is no guarantee that you will get a 200 and text/html content. You should check the response code and only cache if you did indeed get a 200.

Need to transfer multiple files from client to server

I'm recently working on a project in which I'm basically making a dropbox clone. The server and client are working fine but I'm having a slight issue. I'm able to transfer a single file from the client to the server but when I try to transfer all the files together it gives me an error after the transfer of the first file so basically my code is only working for a single file. I need to make it work for multiple files. Any help will be appreciated. Here's my code
Server Code
import socket
import thread
import hashlib
serversock = socket.socket()
host = socket.gethostname();
port = 9000;
serversock.bind((host,port));
filename = ""
serversock.listen(10);
print "Waiting for a connection....."
clientsocket,addr = serversock.accept()
print("Got a connection from %s" % str(addr))
while True:
size = clientsocket.recv(1)
filesz = clientsocket.recv(1)
if filesz.isdigit():
size += filesz
filesize = int(size)
else:
filesize = int(size)
print filesize
for i in range(0,filesize):
if filesz.isdigit():
filename += clientsocket.recv(1)
else:
filename += filesz
filesz = "0"
print filename
file_to_write = open(filename, 'wb')
while True:
data = clientsocket.recv(1024)
#print data
if not data:
break
file_to_write.write(data)
file_to_write.close()
print 'File received successfully'
serversock.close()
Client Code
import socket
import os
import thread
s = socket.socket()
host = socket.gethostname()
port = 9000
s.connect((host, port))
path = "C:\Users\Fahad\Desktop"
directory = os.listdir(path)
for files in directory:
print files
filename = files
size = bytes(len(filename))
#print size
s.send(size)
s.send(filename)
file_to_send = open(filename, 'rb')
l = file_to_send.read()
s.sendall(l)
file_to_send.close()
print 'File Sent'
s.close()
Here's the error that I get
Waiting for a connection.....
Got a connection from ('192.168.0.100', 58339)
13
Betternet.lnk
File received successfully
Traceback (most recent call last):
File "server.py", line 22, in <module>
filesize = int(size)
ValueError: invalid literal for int() with base 10: ''

There are several minor issues in your snippet. Maybe you could do something like this?
import socket
import thread
import hashlib
serversock = socket.socket()
host = socket.gethostname();
port = 9000;
serversock.bind((host,port));
filename = ""
serversock.listen(10);
print "Waiting for a connection....."
clientsocket,addr = serversock.accept()
print("Got a connection from %s" % str(addr))
while True:
size = clientsocket.recv(16) # Note that you limit your filename length to 255 bytes.
if not size:
break
size = int(size, 2)
filename = clientsocket.recv(size)
filesize = clientsocket.recv(32)
filesize = int(filesize, 2)
file_to_write = open(filename, 'wb')
chunksize = 4096
while filesize > 0:
if filesize < chunksize:
chunksize = filesize
data = clientsocket.recv(chunksize)
file_to_write.write(data)
filesize -= len(data)
file_to_write.close()
print 'File received successfully'
serversock.close()
Client:
import socket
import os
import thread
s = socket.socket()
host = socket.gethostname()
port = 9000
s.connect((host, port))
path = "blah"
directory = os.listdir(path)
for files in directory:
print files
filename = files
size = len(filename)
size = bin(size)[2:].zfill(16) # encode filename size as 16 bit binary
s.send(size)
s.send(filename)
filename = os.path.join(path,filename)
filesize = os.path.getsize(filename)
filesize = bin(filesize)[2:].zfill(32) # encode filesize as 32 bit binary
s.send(filesize)
file_to_send = open(filename, 'rb')
l = file_to_send.read()
s.sendall(l)
file_to_send.close()
print 'File Sent'
s.close()
Here the client also sends the size of the file. Both size and filesize are encoded as binary strings (you could do another method). The filename length (size) can take values up to 2^16 and the send file can have up to 2^32 byte (that is 2^filesize).

I believe you're actually receiving all the files' content but then writing them all to one file.
Your server is only accepting a single connection and it writes whatever data it receives into the file until it receives no more data. That won't happen until the client closes its socket at the end.
There are a couple of ways to fix this.
Move the accept call into the server loop and the connect call into the client loop. Have your client connect, send the filename, transfer the entire content of a single file, then close the connection. On the next iteration, do it all over again.
Alternatively, at the beginning of each file, have the client send to the server both the filename to be transferred and the file size (so the server knows how to find the end of the file content). Then write exactly that many bytes to the server. (But see also below as to transmitting the file size.)
I would recommend (1) as more robust and easier to implement.
Second subject: your mechanism for sending the filename is flawed. If I follow it correctly, your program would not work properly if the filename being transmitted begins with a digit as the server will not be able to determine how many bytes are being used to send the filename length. There are two usual ways of sending numbers:
Use the struct module to format a binary integer in a well-defined way. You actually send the "packed" format, and the server would unpack it. It would then know exactly how many bytes long to receive for the filename.
Just send a header line containing the filename followed by a null byte or newline (or some other well-defined defined terminator byte). Then the server can read a byte at a time of the filename until it sees the terminator.

You could create multiple sockets and write using makefile:
server:
import socket
import threading
import time
serversock = socket.socket()
host = socket.gethostname()
port = 9000
serversock.bind((host, port))
serversock.listen(10)
print("Waiting for a connection.....")
def reader(client):
fle = client.makefile('r')
filename = fle.readline()
client.send("Got file {}\n".format(filename))
file_to_write = open(filename.rstrip(), 'wb')
client.send("Starting writing {}\n".format(filename))
file_to_write.write(fle.read())
file_to_write.close()
client.send("Finished writing {}\n".format(filename))
while True:
client, addr = serversock.accept()
print("Got a connection from %s" % str(addr))
client_serve_thread = threading.Thread(target=reader, args=tuple((client,)))
client_serve_thread.start()
time.sleep(0.001)
serversock.close()
client:
import socket
import os
import thread
import os
host = socket.gethostname()
port = 9000
path = "/home/padraic/t"
directory = os.listdir(path)
for file in directory:
s = socket.socket()
s.connect((host, port))
filename = os.path.join(path, file)
s.send(file+"\n")
print(s.recv(1024))
file_to_send = open(os.path.join(path, file), 'rb')
s.send(file_to_send.read())
print('File Sent\n')
file_to_send.close()
rec = s.recv(1024)
print(rec)
s.close()

Python IO error all the time attempting to open a file

From command line
client.py Aaron 12000 HelloWorld.html GET
client.py
def main(argv):
serverName = argv[0]
serverPort = int(argv[1])
fileName = argv[2]
typeOfHttpRequest = argv[3]
clientSocket = socket(AF_INET, SOCK_STREAM)
clientSocket.connect((serverName, serverPort))
clientSocket.send(typeOfHttpRequest + " " + fileName + " HTTP/1.1\r\n\r\n")
content = clientSocket.recv(1024)
print content
clientSocket.close()
if __name__ == "__main__":
main(sys.argv[1:])
server.py
while True:
#Establish the connection
print 'Ready to serve....'
connectionSocket, addr = serverSocket.accept()
try:
message = connectionSocket.recv(1024)
typeOfRequest = message.split()[0]
filename = message.split()[1]
print typeOfRequest
print filename
f = open(filename[1:])
outputdata = f.read()
if typeOfRequest == 'GET':
for i in range(0, len(outputdata)):
connectionSocket.send(outputdata[i])
connectionSocket.close()
elif typeOfRequest == 'HEAD':
connectionSocket.send(True)
except IOError:
connectionSocket.send('HTTP/1.1 404 Not Found')
connectionSocket.close()
serverSocket.close()
I have put HelloWorld.html in the same directory as server.py but this always generates an IOError. Anyone know why it might be the case?
The files are located in C:\Networking
os.getcwd shows C:\Networking
HelloWorld.html is located in C:/networking/HelloWorld.html
Filename prints out correctly.

As you might have noticed, you were trying to strip the / from the beginning of the URL, though it was not there. However, there are other errors in your code, which mean that it does not work like a HTTP server:
First of all, recv() is not guaranteed to read all the data - even if there would be total of 1024 bytes written to a socket, recv(1024) could return just 10 bytes, say. Thus it is better to do in a loop:
buffer = []
while True:
data = connection_socket.recv(1024)
if not data:
break
buffer.append(data)
message = ''.join(buffer)
Now message is guaranteed to contain everything.
Next, to handle the header lines of the request, you can use
from cStringIO import StringIO
message_reader = StringIO(message)
first_line = next(message_reader)
type_of_request, filename = message.split()[:2]
With this it is easier to extend your code for more complete HTTP support.
Now open the file with open, with with statement:
with open(filename) as f:
output_data = f.read()
This ensures that the file is closed properly too.
Finally, when you respond to the request, you should answer with HTTP/1.0, not HTTP/1.1 as you are not supporting the full extent of HTTP/1.1. Also, even an OK response needs to respond with full headers, say with:
HTTP/1.1 200 OK
Server: My Python Server
Content-Length: 123
Content-Type: text/html;charset=UTF-8
data goes here....
Thus your send routine should do that:
if typeOfRequest == 'GET':
headers = ('HTTP/1.0 200 OK\r\n'
'Server: My Python Server\r\n'
'Content-Length: %d\r\n'
'Content-Type: text/html;charset=UTF-8\r\n\r\n'
'Connection: close\r\n'
) % len(output_data)
connection_socket.sendall(headers)
connection_socket.sendall(output_data)
Notice how you can use sendall to send all data from a string.

python server with images

I'm running an python server that loads a login page. All the html pages will load (they're in the same folder as the html page that's being hosted, so are the images), but the images that are in the same folder as the html don't load.Anybody know what's wrong with the image loading ? Thanks. Here's the code:
#/usr/bin/python
from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
from os import curdir, sep
PORT_NUMBER = 8080
#This class will handles any incoming request from
#the browser
class myHandler(BaseHTTPRequestHandler):
#Handler for the GET requests
def do_GET(self):
if self.path=="/":
self.path="login.html"
try:
#Check the file extension required and
#set the right mime type
sendReply = False
if self.path.endswith(".html"):
mimetype='text/html'
sendReply = True
if self.path.endswith(".jpg"):
mimetype='image/jpg'
sendReply = True
if self.path.endswith(".gif"):
mimetype='image/gif'
sendReply = True
if self.path.endswith(".js"):
mimetype='application/javascript'
sendReply = True
if self.path.endswith(".css"):
mimetype='text/css'
sendReply = True
if sendReply == True:
#Open the static file requested and send it
f = open(curdir + sep + self.path)
self.send_response(200)
self.send_header('Content-type',mimetype)
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except IOError:
self.send_error(404,'File Not Found: %s' % self.path)
try:
#Create a web server and define the handler to manage the
#incoming request
server = HTTPServer(('', PORT_NUMBER), myHandler)
print 'Started httpserver on port ' , PORT_NUMBER
#Wait forever for incoming htto requests
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down the web server'
server.socket.close()

You can try the following code for jpg/png files accordingly
if self.path.endswith(".jpg"):
f = open(applicationPath + '/' + self.path, 'rb')
self.send_response(200)
self.send_header('Content-type', 'image/png')
self.end_headers()
self.wfile.write(f.read())
f.close()
return

You have to add mode='rb' to f = open(curdir + sep + self.path) because you are reading a binary file.