Develop Reference

Creating a simple HTTP python server to handle broken pipe

I create a simple python http server and I want it to display only files from a directory I want to, thus changing always returning "Hello world!" and also how could I handle the broken pipe errors? I tried to do a try catch there but I'm not sure if it's working:
#!/usr/bin/python
from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
PORT_NUMBER = 8089
#This class will handles any incoming request from
#the browser
class myHandler(BaseHTTPRequestHandler):
#Handler for the GET requests
def do_GET(self):
self.send_response(200)
self.send_header('Content-type','text/html')
self.end_headers()
self.wfile.write("Hello World !")
return
try:
#Create a web server and define the handler to manage the
#incoming request
server = HTTPServer(('', PORT_NUMBER), myHandler)
print 'Started http server on port ' , PORT_NUMBER
#Wait forever for incoming http requests
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down the web server'
server.socket.close()
except socket.error:
pass
This is my Error:
someIP - - [25/Dec/2019 09:17:11] "GET someFILE HTTP/1.0" 200 -
Traceback (most recent call last):
File "/usr/lib/python2.7/SocketServer.py", line 293, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 321, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.7/SocketServer.py", line 657, in __init__
self.finish()
File "/usr/lib/python2.7/SocketServer.py", line 716, in finish
self.wfile.close()
File "/usr/lib/python2.7/socket.py", line 283, in close
self.flush()
File "/usr/lib/python2.7/socket.py", line 307, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe

Example which follows is the basic example of creating basic web server for serving the static files. Additional comments you can find inside the code, with one additional note: 403 Forbidden implementation can be replaced with file indexing page, for what you need to do the additional generation of it (according to your question, this is out of scope for now.)
from http.server import HTTPServer, BaseHTTPRequestHandler
from os import curdir, sep, path
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200) #Request received, sending OK status
self.end_headers()
try:
if(path.isdir(self.path)): #Checking if user requested access to dirrectory (not to a particular file)
self.send_response(403)
self.wfile.write(str.encode("Listing of directories not permited on this server")) #Preventing directory listing, in case you dont want to allow file indexing.
else: #If user is requesting access to a file, file content is read and displayed.
f = open(curdir + sep + self.path, 'rb')
self.wfile.write(f.read())
f.close()
except IOError: #Covering the 404 error, in case user requested non-existing file
print("File "+self.path+" not found")
self.send_response(404)
httpd = HTTPServer(('localhost', 8000), SimpleHTTPRequestHandler)
httpd.serve_forever()

I used to get the same error earlier
My Code :
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE,SIG_DFL)
Paste it in the end and it should work!

int treated as `self' in python3

I want to build a simple server that dispatches requests depending on their path. So I wrote the following code:
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import socket
class Handler:
"""处理对应资源的请求"""
def __init__(self, dispatcher):
self.dispatcher = dispatcher
def handle(self):
"""根据dispatcher解析出来的path，调用对应的方法"""
command = 'do_' + self.dispatcher.command
print(command)
if not hasattr(self, command):
return;
method = getattr(self, command)
method(self)
def do_GET(self):
response = {
'message': 'message'
}
dispatcher.protocol_version = 'HTTP/1.1'
dispatcher.send_response(200, 'OK')
dispatcher.send_header('Content-type', 'text/plain')
dispatcher.wfile.write(bytes(json.dumps(response), 'utf-8'))
class dispatcher(BaseHTTPRequestHandler):
"""
根据url的不同分发请求
"""
def handle_one_request(self):
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG)
return
if not self.raw_requestline:
self.close_connection = True
return
if not self.parse_request():
# An error code has been sent, just exit
return
print(self.command)
print(self.path)
if self.path.startswith('/wrong'):
Handler(self).handle()
self.wfile.flush() #actually send the response if not already done.
except socket.timeout as e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = True
return
if __name__ == '__main__':
server = HTTPServer(('', 8080), dispatcher)
server.serve_forever()
The dispatcher will parse the incoming request and get the path so that it can decide which handler to call(though there is only one here for now).
In the handler class, it will call corresponding method based on the http method. In the do_GET method, it will call some methods in the dispatcher and that's where things go wrong.
When I ran this program and execute curl http://localhost:8080/wrong, I had the following exception:
GET
/wrong
do_GET
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 50072)
Traceback (most recent call last):
File "/usr/lib/python3.5/socketserver.py", line 313, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 341, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 354, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python3.5/socketserver.py", line 681, in __init__
self.handle()
File "/usr/lib/python3.5/http/server.py", line 422, in handle
self.handle_one_request()
File "hello.py", line 51, in handle_one_request
Handler(self).handle()
File "hello.py", line 18, in handle
method()
File "hello.py", line 25, in do_GET
dispatcher.send_response(200, 'OK')
File "/usr/lib/python3.5/http/server.py", line 487, in send_response
self.log_request(code)
AttributeError: 'int' object has no attribute 'log_request'
----------------------------------------
log_request is defined as follows in the super class of dispatcher:
def log_request(self, code='-', size='-'):
# blablabla
and it is called in dispatcher.send_response.
I guess the problem is that the 200 is treated as self by the interpreter.
If my guess is right, why is this happening?
If not, what causes this exception?
I know this question is still in the grammar level, but I'd appreciate it if someone can help me.

Getting error: [Errno 10053] while trying to send a file in http response

Am trying to send a big file in http response by writing into wfile variable of BaseHTTPRequestHandler in Python, when I am trying to do that I am ending with the below exception in my Python code always.
error: [Errno 10053] An established connection was aborted by the software in your machine
Can any one help me to resolve this?? why am getting the error?
If the way am sending large file in HTTP response is not the good one, please suggest where I can refer.
Thanks in advance!!!
import os
import urlparse
import BaseHTTPServer
from SocketServer import ThreadingMixIn
import urlparse
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def handle(self):
BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
def sendError(self, errorCode, errorMessage):
self.send_response(errorCode, errorMessage)
self.send_header("Content-type", "text/plain")
self.send_header("Content-Length", str(len(errorMessage)))
self.end_headers()
self.wfile.write(errorMessage)
def do_GET(self):
scm, netloc, path, params, query, fragment = urlparse.urlparse(self.path, 'http')
if path.find(".ld") > 0:
filename = path.rpartition("/")[2]
try:
with open(filename, 'rb') as f:
self.send_response(200, "Ok")
self.send_header("Content-type","application/octet-stream")
total_size = os.path.getsize(filename)
self.send_header("Content-Length", total_size)
self.end_headers()
self.wfile.write(f.read())
except IOError:
self.sendError(404, "Not Found")
class ThreadedHTTPServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
def main():
Handler.close_connection = 0
Handler.protocol_version = 'HTTP/1.1'
global httpd
httpd = ThreadedHTTPServer(("", 8900), Handler)
httpd.daemon_threads = True
httpd.serve_forever()
if __name__ == "__main__":
main()
Error Trace:
Exception happened during processing of request from ('172.24.128.21', 19418)
Traceback (most recent call last):
File "C:\Python27\lib\SocketServer.py", line 593, in process_request_thread
self.finish_request(request, client_address)
File "C:\Python27\lib\SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "C:\Python27\lib\SocketServer.py", line 649, in __init__
self.handle()
File "simple.py", line 10, in handle
BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
File "C:\Python27\lib\BaseHTTPServer.py", line 342, in handle
self.handle_one_request()
File "C:\Python27\lib\BaseHTTPServer.py", line 310, in handle_one_request
self.raw_requestline = self.rfile.readline(65537)
File "C:\Python27\lib\socket.py", line 476, in readline
data = self._sock.recv(self._rbufsize)
error: [Errno 10054] An existing connection was forcibly closed by the remote ho
st

OK, so I tried your code after cleaning it up a bit:
def do_GET(self):
_, _, path, _, _, _ = urlparse.urlparse(self.path, 'http')
if path.find(".ld") > 0:
filename = path.rpartition("/")[2]
try:
with open(filename, 'rb') as f:
self.send_response(200, "Ok")
self.send_header("Content-type", self.headers.getheader("Content-type", ""))
total_size = os.path.getsize(filename)
self.send_header("Content-Length", total_size)
self.end_headers()
self.wfile.write(f.read())
except IOError:
self.sendError(404, "Not Found")
I put a foo.pl file in the same folder; then, when doing curl http://localhost/foo.pl, I get a nice response with the content of the file, and no errors whatsoever.
I must also say that it looks like you're just trying to get the file name without the path part using path.rpartition("/")[2], but for that, you should just use os.path.basename:
>>> os.path.basename('foo/bar/baz.pl')
'baz.pl'
Also, path.find(".ld") > 0 should probably be path.endswith(".ld") instead.
EDIT: to support large files (efficiently):
CHUNK_SIZE = 1024 * 100 # 100kB chunks
while True:
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
self.wfile.write(chunk)

Apache bench and python BaseHTTPServer broken pipe exception

I have a simple HTTP server implemented in python as follows:
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write('Hello World!')
if __name__ == '__main__':
try:
server = HTTPServer(('', 8000), Handler)
print 'Listening on 0.0.0.0:8000'
server.serve_forever()
except KeyboardInterrupt:
print '^C, halting'
server.socket.close()
Then I run apache bench with concurrency level of 50:
ab -v 3 -c 50 -n 50000 http://localhost:8000/
I started getting this error in apache bench:
apr_socket_recv: Connection reset by peer (104)
and on the HTTP server console I see this:
Exception happened during processing of request from ('127.0.0.1', 51545)
Traceback (most recent call last):
File "/usr/lib/python2.6/SocketServer.py", line 281, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 307, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 320, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.6/SocketServer.py", line 615, in __init__
self.handle()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 329, in handle
self.handle_one_request()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 323, in handle_one_request
method()
File "./test.py", line 12, in do_GET
self.send_response(200)
File "/usr/lib/python2.6/BaseHTTPServer.py", line 384, in send_response
self.send_header('Server', self.version_string())
File "/usr/lib/python2.6/BaseHTTPServer.py", line 390, in send_header
self.wfile.write("%s: %s\r\n" % (keyword, value))
File "/usr/lib/python2.6/socket.py", line 300, in write
self.flush()
File "/usr/lib/python2.6/socket.py", line 286, in flush
self._sock.sendall(buffer)
error: [Errno 32] Broken pipe
Also I notice that the HTTPServer gets real sluggish
I have tried to increase the request_queue_size of HTTPServer by doing this:
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
class MyServer(HTTPServer):
request_queue_size = 100
timeout = None
class Handler(BaseHTTPRequestHandler):
counter = 0
def do_GET(self):
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(str(self.counter))
Handler.counter += 1
if __name__ == '__main__':
try:
server = MyServer(('', 8000), Handler)
print 'Listening on 0.0.0.0:8000'
server.serve_forever()
except KeyboardInterrupt:
print '^C, halting'
server.socket.close()
but I still get the broken pipe error at the end of the test run once in a while. If I increase request_queue_size to a huge number then the error goes away. I don't understand why because the Apache bench -c value is 50 and request_queue_size of 100 should be sufficient. Why is it that the request_queue_size needs to be much bigger than the Apache bench concurrency setting? Does Apache bench fire off more connections even if the previous one hasn't finished and hence the total number of concurrent connections can be more than the -c setting?

This is a problem with apachebench, but not with httperf. I'm not really sure what the difference in they way they handle the connection is.
Increase from default(5) to 1000. I haven't noticed any huge blowups in memory and performance has increased dramatically, while eliminating errors.

HTTP server hangs while accepting packets

I have written a simple http server to handle POST requests:
class MyHandler( BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST( self ):
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
postvars = {}
try:
if ctype == 'application/x-www-form-urlencoded':
length = int(self.headers.getheader('content-length'))
postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
self.send_response( 200 )
self.send_header( "Content-type", "text")
self.send_header( "Content-length", str(len(body)) )
self.end_headers()
self.wfile.write(body)
except:
print "Error"
def httpd(handler_class=MyHandler, server_address = ('2.3.4.5', 80)):
try:
print "Server started"
srvr = BaseHTTPServer.HTTPServer(server_address, handler_class)
srvr.serve_forever() # serve_forever
except KeyboardInterrupt:
server.socket.close()
if __name__ == "__main__":
httpd( )
The server runs fine but sometimes it just hangs. When I press CTRL+C it gives the following error and then continues receiving data:
Exception happened during processing of request from ('1.1.1.2', 50928)
Traceback (most recent call last):
File "/usr/lib/python2.6/SocketServer.py", line 281, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 307, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 320, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.6/SocketServer.py", line 615, in __init__
self.handle()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 329, in handle
self.handle_one_request()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 312, in handle_one_request
self.raw_requestline = self.rfile.readline()
File "/usr/lib/python2.6/socket.py", line 406, in readline
data = self._sock.recv(self._rbufsize)
KeyboardInterrupt
Can someone tell me how to correct this? I can't make sense of the errors.

I completely rewrote my solution in order to fix two disadvantages:
It can treat timeout even if client has opened only a connection but did not start communicate.
If the client opens a connection, a new server process is forked. A slow client can not block other.
It is based on your code as most as possible. It is now a complete independent demo. If you open a browser on http://localhost:8000/ and write 'simulate error' to form and press submit, it simulates runtime error.
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ForkingMixIn
from urllib.parse import parse_qs
class MyHandler(BaseHTTPRequestHandler):
def do_POST(self):
ctype, pdict = self.headers.get_params(header='content-type')[0]
postvars = {}
try:
if ctype == 'application/x-www-form-urlencoded':
length = int(self.headers['content-length'])
postvars = parse_qs(self.rfile.read(length), keep_blank_values=1
encoding='utf-8')
assert postvars.get('foo', '') != ['bar'] # can simulate error
body = 'Something\n'.encode('ascii')
self.send_response(200)
self.send_header("Content-type", "text")
self.send_header("Content-length", str(len(body)))
self.end_headers()
self.wfile.write(body)
except Exception:
self.send_error(500)
raise
def do_GET(self):
# demo for testing POST by web browser - without valid html
body = ('Something\n<form method="post" action="http://%s:%d/">\n'
'<input name="foo" type="text"><input type="submit"></form>\n'
% self.server.server_address).encode('ascii')
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-length", str(len(body)))
self.end_headers()
self.wfile.write(body)
class ForkingHTTPServer(ForkingMixIn, HTTPServer):
def finish_request(self, request, client_address):
request.settimeout(30)
# "super" can not be used because BaseServer is not created from object
HTTPServer.finish_request(self, request, client_address)
def httpd(handler_class=MyHandler, server_address=('localhost', 8000)):
try:
print("Server started")
srvr = ForkingHTTPServer(server_address, handler_class)
srvr.serve_forever() # serve_forever
except KeyboardInterrupt:
srvr.socket.close()
if __name__ == "__main__":
httpd()
Forking can be disabled for e.g. debugging purposes by removing class SocketServer.ForkingMixIn from code.
EDIT Updated for Python 3, but a warning from Python 3 docs should be noted:
Warning: http.server is not recommended for production. It only implements basic security checks.