Python SSL Socket Server - python

I want to set up a basic ssl-authenticated socket server to do some network communication. I'm getting the error below. It seems to be coming from the SSLIOStream not handshaking before reading:
File "simple_ssl_server.py", line 70, in connection_ready
node_io_stream.read_until("OK", on_ok)
File "/home/tombrown/skyhook/lib/python2.7/site-packages/tornado-2.1.1-py2.7.egg/tornado/iostream.py", line 161, in read_until
if self._read_to_buffer() == 0:
File "/home/tombrown/skyhook/lib/python2.7/site-packages/tornado-2.1.1-py2.7.egg/tornado/iostream.py", line 375, in _read_to_buffer
chunk = self._read_from_socket()
File "/home/tombrown/skyhook/lib/python2.7/site-packages/tornado-2.1.1-py2.7.egg/tornado/iostream.py", line 635, in _read_from_socket
chunk = self.socket.read(self.read_chunk_size)
File "/usr/lib/python2.7/ssl.py", line 151, in read
return self._sslobj.read(len)
SSLError: [Errno 1] _ssl.c:1354: error:1408F044:SSL routines:SSL3_GET_RECORD:internal error
Here's my server code:
import tornado.web
import tornado.httpserver
import select
import socket
import tornado.iostream
import random
import logging
import ssl
import functools
class SSLSocketServer(object):
def __init__(self, io_loop=None, config_file=None, debug=False):
if io_loop is None: io_loop = tornado.ioloop.IOLoop.instance()
# Set up our node-accepting socket on port 8013
HOST = '' # Symbolic name meaning all available interfaces
PORT = 8013 # Arbitrary non-privileged port
server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_sock.setblocking(0)
server_sock.bind((HOST, PORT))
# We allow a backlog of up to 128 pending connections.
server_sock.listen(128)
callback = functools.partial(self.connection_ready, server_sock)
io_loop.add_handler(server_sock.fileno(),
callback, io_loop.READ)
def connection_ready(self, sock, fd, events):
# In part from: https://github.com/saucelabs/monocle/blob/7bd978f1c6a2ad3d78dd3da0b5b73c3e215ebbf3/monocle/tornado_stack/network/__init__.py
while True:
# Wait for the basic socket to be available.
try:
node_sock, address = sock.accept()
except socket.error, e:
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
raise
return
# Wait for the ssl socket to be available.
try:
node_sock = ssl.wrap_socket(node_sock,
do_handshake_on_connect=False,
server_side=True,
certfile="cert.pem",
ssl_version=ssl.PROTOCOL_TLSv1)
except ssl.SSLError, err:
if err.args[0] == ssl.SSL_ERROR_EOF:
s.close()
return
else:
raise
except socket.error, err:
if err.args[0] == errno.ECONNABORTED:
s.close()
return
else:
raise
node_io_stream = tornado.iostream.SSLIOStream(node_sock)
def on_ok():
print "recieved OK!"
node_io_stream.read_until("OK", on_ok)
if __name__ == '__main__':
# Get a handle to the instance of IOLoop
io_loop = tornado.ioloop.IOLoop.instance()
worker = SSLSocketServer(io_loop)
# Start the IOLoop
io_loop.start()
And here's the client code:
import sys
import logging
import socket
from tornado import iostream
from tornado import ioloop
import uuid
from tornado.options import define, options
import json
import ssl
def main():
delim = '\r\n\r\n'
def send_request():
print "sending OK"
stream.write("OK")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
# stream = iostream.IOStream(s)
stream = iostream.SSLIOStream(s,
ssl_options= dict(
ca_certs="fake_auth/server_certfile.pems",
cert_reqs=ssl.CERT_NONE))
print "about to connect"
stream.connect(('', 8013), send_request)
ioloop.IOLoop.instance().start()
if __name__ == '__main__':
main()
I created the keyfile and certificate with the following command:
openssl req -new -x509 -days 365 -nodes -out cert.pem -keyout cert.pem

Tornado is able to maintain an SSL connection itself using python 2.6+ and OpenSSL. Why are you attempting to manually build out the SSL socket connection?
Check out: http://www.tornadoweb.org/documentation/httpserver.html#http-server
Key Caption:
HTTPServer can serve SSL traffic with Python 2.6+ and OpenSSL. To make
this server serve SSL traffic, send the ssl_options dictionary
argument with the arguments required for the ssl.wrap_socket method,
including “certfile” and “keyfile”:
HTTPServer(applicaton, ssl_options={
"certfile": os.path.join(data_dir, "mydomain.crt"),
"keyfile": os.path.join(data_dir, "mydomain.key"),
})

pems? Shouldn't this be pem?
ca_certs="fake_auth/server_certfile.pems",
Did you get it to work?

Related

How can I create a Python SSL Client/Server pair where only the server authenticates the client

I have been trying to get a simple Python SSL example working for a day now with no luck. I want to create an SSL server and SSL client. The server should authenticate the client. The Python docs are pretty light on examples for the SSL module, and in general I can't find many working examples. The code I am working with is as follows;
Client:
import socket
import ssl
class SSLClient:
def __init__(self, server_host, server_port, client_cert, client_key):
self.server_host = server_host
self.server_port = server_port
self._context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
self._context.load_cert_chain(client_cert, client_key)
self._sock = None
self._ssock = None
def __del__(self):
self.close()
def connect(self):
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(
self._sock, server_hostname=self.server_host
)
self._ssock.connect((self.server_host, self.server_port))
def send(self, msg):
self._ssock.send(msg.encode())
def close(self):
self._ssock.close()
Server:
import socket
import ssl
from threading import Thread
class SSLServer:
def __init__(self, host, port, cafile, chunk_size=1024):
self.host = host
self.port = port
self.chunk_size = chunk_size
self._context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
self._context.load_verify_locations(cafile)
self._ssock = None
def __del__(self):
self.close()
def connect(self):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
sock.bind((self.host, self.port))
sock.listen(5)
with self._context.wrap_socket(sock, server_side=True) as self._ssock:
conn, _ = self._ssock.accept()
while True:
data = conn.recv(self.chunk_size).decode()
print(data)
if data is None:
break
def close(self):
self._ssock.close()
class SSLServerThread(Thread):
def __init__(self, server):
super().__init__()
self._server = server
self.daemon = True
def run(self):
self._server.connect()
def stop(self):
self._server.close()
Test script:
import client, server
from os import path
from time import sleep
server_host = "localhost"
server_port = 11234
client_cert = path.join(path.dirname(__file__), "client.crt")
client_key = path.join(path.dirname(__file__), "client.key")
s = server.SSLServer(server_host, server_port, client_cert)
s_thread = server.SSLServerThread(s)
s_thread.start()
sleep(2)
c = client.SSLClient(server_host, server_port, client_cert, client_key)
c.connect()
c.send("This is a test message!")
c.close()
s.close()
I generated my client certificate and key using the following command:
openssl req -newkey rsa:2048 \
-x509 \
-sha256 \
-days 3650 \
-nodes \
-out client.crt \
-keyout client.key \
-subj "/C=UK/ST=Scotland/L=Glasgow/O=Company A/OU=Testing/CN=MyName"
The test script seems to start the server and allow the client to connect, but I am getting a BrokenPipeError when I try to send the test message.
Annoyingly I have been getting various different error messages as I go, so it's likely a combination of things. This is a simple example I created to try and get something working. On my more complex example I get "NO_SHARED_CIPHERS" when the client attempts to connect to the server. Annoyingly I can't see why this simple example seems to get further than the more complex one (i.e. the connection seems to be established successfully) even though they are set up almost identically.
I have uploaded a repo at git#github.com:stevengillies87/python-ssl-client-auth-example.git if anyone would like to test it.
I realised the first bug came from copy pasting and example and not realising how it differed from my code in its setup. It used socket.socket() to create the socket whereas my example used socket.create_connection(), which also connects the socket. This was the reason I was getting a BrokenPipeError. Now both my simple example and the actual code I am writing both have a consistent NO_SHARED_CIPHER error. I added a line to the source code to connect the client after the socket has been wrapped.
So, as expected it was a combination of things.
Before I added the SSL layer to my code it worked with TCP sockets. I was using socket.create_connection() in the client to create and connect a socket in one call. When I added SSL I continued to do this but because I was attempting to connect to an SSL server via a TCP socket I was getting a NO_SHARED_CIPHER error.
The solution to this problem was to only create the TCP socket with sock = socket.socket(), wrap it with ssock = ssl_context.wrap_context(sock) and then call connect on the SSL layer, ssock.connect((host, port)).
However, I was still getting a handshaking error on connection. I found this link, https://www.electricmonk.nl/log/2018/06/02/ssl-tls-client-certificate-verification-with-python-v3-4-sslcontext/, which provided a detailed example of how to create mutually authenticating SSL client/server. Crucially, the author pointed out that hostname used for server authentication must match the "common name" entered when creating the server.crt and server.key files. Previously I had just been using the same host that I was connecting to, "localhost" in this case. They also noted that the ssl_context verify mode should be set to verify_mode = ssl.CERT_REQUIRED for client auth.
Once the example worked I set about removing the client auth of the server. This was done by changing the client SSL context from ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) to ssl.SSLContext(). The client now does not require the server.crt file to connect successfully.
Frustratingly I still need to create server cert/key files and load them into the server using ssl_context.load_cert_chain(), even though I do not need the server to be authenticated. If I try to remove this step from the server I get a NO_SHARED_CIPHER error again. If anyone knows how I can avoid this then please let me know, or explain why it is necessary.
Working code below, and updated at the github link in the question.
Client:
import socket
import ssl
class SSLClient:
def __init__(
self, server_host, server_port, sni_hostname, client_cert, client_key,
):
self.server_host = server_host
self.server_port = server_port
self.sni_hostname = sni_hostname
self._context = ssl.SSLContext()
self._context.load_cert_chain(client_cert, client_key)
self._sock = None
self._ssock = None
def __del__(self):
self.close()
def connect(self):
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock,)
self._ssock.connect((self.server_host, self.server_port))
def send(self, msg):
self._ssock.send(msg.encode())
def close(self):
self._ssock.close()
Server:
import socket
import ssl
from threading import Thread
class SSLServer:
def __init__(
self, host, port, server_cert, server_key, client_cert, chunk_size=1024
):
self.host = host
self.port = port
self.chunk_size = chunk_size
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(server_cert, server_key)
self._context.load_verify_locations(client_cert)
def connect(self):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
sock.bind((self.host, self.port))
sock.listen(5)
while True:
conn, _ = sock.accept()
with self._context.wrap_socket(conn, server_side=True) as sconn:
self._recv(sconn)
def _recv(self, sock):
while True:
data = sock.recv(self.chunk_size)
if data:
print(data.decode())
else:
break
class SSLServerThread(Thread):
def __init__(self, server):
super().__init__()
self._server = server
self.daemon = True
def run(self):
self._server.connect()
Test:
import client, server
from os import path
from time import sleep
server_host = "127.0.0.1"
server_port = 35689
server_sni_hostname = "www.company-b.com"
client_cert = path.join(path.dirname(__file__), "client.crt")
client_key = path.join(path.dirname(__file__), "client.key")
server_cert = path.join(path.dirname(__file__), "server.crt")
server_key = path.join(path.dirname(__file__), "server.key")
s = server.SSLServer(server_host, server_port, server_cert, server_key, client_cert)
s_thread = server.SSLServerThread(s)
s_thread.start()
sleep(2)
c = client.SSLClient(
server_host, server_port, server_sni_hostname, client_cert, client_key
)
c.connect()
c.send("This is a test message!")
c.close()

Not able to lock the thread in socket using Multiprocess

I have gone through the link: multiprocessing.Pool - PicklingError: Can't pickle <type 'thread.lock'>: attribute lookup thread.lock failed
Still I did not got the solution for that.
Here is what I have tried:
Server
import multiprocessing
import socket
from iqoptionapi.stable_api import IQ_Option
import time
def handle(client_socket,address,I_want_money):
print(address)
client_socket.sendall("Happy".encode())
client_socket.close()
return
class Server(object):
def __init__(self, hostname, port):
# import logging
# self.logger = logging.getLogger("server")
self.hostname = hostname
self.port = port
self.I_want_money=IQ_Option("email","password")
self.I_want_money.suspend = 0.1
print("I am ON.........")
def start(self):
# self.logger.debug("listening")
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.bind((self.hostname, self.port))
self.socket.listen(1)
while True:
conn, address = self.socket.accept()
# self.logger.debug("Got connection")
process = multiprocessing.Process(target=handle, args=(conn, address,self.I_want_money,))
process.daemon = True
process.start()
# self.logger.debug("Started process %r", process)
if __name__ == "__main__":
# import logging
# logging.basicConfig(level=logging.DEBUG)
server = Server("0.0.0.0", 9000)
try:
# logging.info("Listening")
server.start()
except Exception as e:
print(e, "\n I had experienced at initialization")
# logging.exception("Unexpected exception")
finally:
# logging.info("Shutting down")
for process in multiprocessing.active_children():
# logging.info("Shutting down process %r", process)
process.terminate()
process.join()
# logging.info("All done")
This is the client:
import socket
if __name__ == "__main__":
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("localhost", 9000))
for i in range(100):
data = "some data"
sock.sendall(data.encode())
result = sock.recv(1024)
print(result)
sock.close()
And finally the error received:
login...
I am ON.........
Can't pickle <class '_thread.lock'>: attribute lookup lock on _thread failed
I had experienced at initialization
Press any key to continue . . . Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Python35\lib\multiprocessing\spawn.py", line 100, in spawn_main
new_handle = steal_handle(parent_pid, pipe_handle)
File "C:\Python35\lib\multiprocessing\reduction.py", line 81, in steal_handle
_winapi.PROCESS_DUP_HANDLE, False, source_pid)
OSError: [WinError 87] The parameter is incorrect
Kindly. let me know the solution for this sort of situation. I am using Python 3.5.0
The IQOption API object probably owns a thread, which can't be passed through to subprocesses. Use threads for concurrency instead...
Here's a roughly equivalent (untested) version of your server code using the built-in socketserver module, using threads (with ThreadingMixIn) to parallelize each client connection.
from iqoptionapi.stable_api import IQ_Option
import socketserver
I_want_money = IQ_Option("email", "password")
I_want_money.suspend = 0.1
class RequestHandler(socketserver.BaseRequestHandler):
def handle(self):
print(self.client_address)
request.sendall("Happy".encode())
request.close()
server = socketserver.ThreadingTCPServer(("0.0.0.0", 9000), RequestHandler)
server.serve_forever()

Handling requests in python SocketServer

I have a code below that does the client-server communication properly.
The client:
# Client
import socket
import pickle
class Model:
def __init__(self, host, port):
self.port = port
self.host = host
def snd_query(self, query):
received_data = []
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((self.host, self.port))
sock.sendall(pickle.dumps(query))
while True:
packet = sock.recv(4096)
if not packet or packet == b'':
break
received_data.append(packet)
try:
content = pickle.loads(b"".join(received_data))
return content
except EOFError:
return None
and a server:
# Server.py
import socketserver
import pickle
import configparser
from data_manipulation import DataManipulation
class MyTCPHandler(socketserver.BaseRequestHandler):
def handle(self):
db = DataManipulation("data.db")
request = pickle.loads(self.request.recv(1024))
if request['command'] == 'GET':
content = db.get_data()
#elif ...:
#... some other logic and database interactions
self.request.sendall(pickle.dumps((content)))
def main(HOST, PORT):
try:
my_server = socketserver.TCPServer((HOST, PORT), MyTCPHandler)
my_server.serve_forever()
except KeyboardInterrupt:
my_server.shutdown()
my_server.server_close()
if __name__ == "__main__":
config = configparser.ConfigParser()
config.read('../params.ini')
main(config['SERVER']['host'], int(config['SERVER']['port']))
In that form with every new handled request a database connection db instance is created.
I would like to avoid it. I would like to make db object ones and raise it with argument read from ini file. Obviously it is enough to read it once.
All examples that I found show simple echo server application. handle() doesn't do much, only prints some stuff.
I am not sure where is the correct place to call db object to have only one instance?
How to properly incorporate some advanced logic in handle() method?

Serve proxy app through sockets Tornado 4.4

I have a little question.
I'm new to the whole socket theory nad the Tornado Framework. Lately I've written a proxy server using the Tornado framework. My app is standing in the middle of a connection client(browser) <===> remote address. So the connection looks like this:
client(my web browser) <=== PROXY(my app) ===> Remote address (stackoverflow.com)
Everything works fine if I use the standard "main" function. But I want to go a little more low - level, I mean I want to create sockets and offer my proxy app through that connection.
My proxy app urls:
# coding: utf-8
"""URL's for proxy app."""
from settings import settings
from tornado.web import (
StaticFileHandler,
url,
)
from handlers import (
mainHandlers,
myProxy,
)
urls = [
url(r"/admin/$", mainHandlers.MainHandler),
url(r"/admin/delete_filter/", mainHandlers.DataDeleteHandler),
url(r"/admin/filters/$", mainHandlers.DataGetter),
url(r"/admin/new_filter/$", mainHandlers.FormHandler),
url(r"/admin/stats/$", mainHandlers.StatsTableHandler),
url(r"/admin/stats/query/$", mainHandlers.AjaxStatsGetHandler),
url(r"/static/", StaticFileHandler, dict(path=settings['static_path'])),
url(r'.*', myProxy.ProxyHandler),
]
My ProxyHandler:
class ProxyHandler(tornado.web.RequestHandler):
SUPPORTED_METHODS = ['GET', 'POST']
def data_received(self, chunk):
pass
def compute_etag(self):
return None # disable tornado Etag
def handle_response(self, response):
if response.error and not isinstance(response.error, tornado.httpclient.HTTPError):
self.set_status(500)
self.write('Internal server error:\n' + str(response.error))
else:
self.set_status(response.code, response.reason)
self._headers = tornado.httputil.HTTPHeaders() # clear tornado default header
for header, v in response.headers.get_all():
if header not in ('Content-Length', 'Transfer-Encoding', 'Content-Encoding', 'Connection'):
self.add_header(header, v) # some header appear multiple times, eg 'Set-Cookie'
secured_page = False
for page in secure_pages:
if page in self.request.uri:
secured_page = True
self.set_header('Content-Length', len(response.body))
self.write(response.body)
break
if response.body and not secured_page:
c.execute('SELECT filter_name FROM filters WHERE filter_type=1')
tags = c.fetchall()
soup = BeautifulSoup(response.body, 'html.parser')
for row in tags:
catched_tags = soup.find_all(str(row[0]))
if catched_tags:
print 'catched: %s of <%s> tags' % (len(catched_tags), str(row[0]))
for tag in catched_tags:
tag.extract()
new_body = str(soup)
self.set_header('Content-Length', len(new_body))
self.write(new_body)
self.finish()
#tornado.web.asynchronous
def get(self):
logger.debug('Handle %s request to %s', self.request.method, self.request.uri)
body = self.request.body
if not body:
body = None
try:
if 'Proxy-Connection' in self.request.headers:
del self.request.headers['Proxy-Connection']
c.execute('SELECT filter_name FROM filters WHERE filter_type=2')
urls = c.fetchall()
for url in urls:
if url[0] in self.request.path:
self.set_status(403)
self.finish()
return
fetch_request(self.request.uri, self.handle_response,
method=self.request.method, body=body, headers=self.request.headers, follow_redirects=False,
allow_nonstandard_methods=True)
except tornado.httpclient.HTTPError as e:
if hasattr(e, 'response') and e.response:
self.handle_response(e.response)
else:
self.set_status(500)
self.write('Internal server error:\n' + str(e))
self.finish()
#tornado.web.asynchronous
def post(self):
return self.get()
The easy main function:
# coding: utf-8
import sys
import tornado.web
from tornado.options import options
from configuration.application import MyApplication
from proxy.urls import proxy_urls
def make_app():
"""Create my application with my settings and urls."""
return MyApplication(proxy_urls)
if __name__ == "__main__":
u"""Main loop."""
app = make_app()
port = options.port
if len(sys.argv) > 1:
port = int(sys.argv[1])
app.listen(port)
print 'tornado working on port %s' % port
tornado.ioloop.IOLoop.current().start()
So I want to change the easy way to the low-level way based on the docs:
import errno
import functools
import tornado.ioloop
import socket
def connection_ready(sock, fd, events):
while True:
try:
connection, address = sock.accept()
except socket.error as e:
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
raise
return
connection.setblocking(0)
handle_connection(connection, address)
if __name__ == '__main__':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(0)
sock.bind(("", port))
sock.listen(128)
io_loop = tornado.ioloop.IOLoop.current()
callback = functools.partial(connection_ready, sock)
io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
io_loop.start()
To accomplish this I read a little about the whole network programming using sockets (https://www.tutorialspoint.com/python/python_networking.htm).
The example in the tutorial works well so I tried to connect the tutorial with the example in the Tornado docs:
# coding: utf-8
import errno
import functools
import socket
import sys
import tornado.httpserver
import tornado.ioloop
import tornado.netutil
import tornado.process
import tornado.web
from tornado.options import options
from configuration.application import MyApplication
def make_app():
u"""Create my application with my settings and urls."""
return MyApplication()
def connection_ready(sock, fd, events):
u"""Function to handle an incoming connection."""
proxy_app = make_app()
server = tornado.httpserver.HTTPServer(proxy_app)
while True:
try:
connection, address = sock.accept()
except socket.error as e:
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
raise
return
print 'Got connection from', address
# connection.setblocking(False)
connection.send(server)
connection.close()
if __name__ == "__main__":
u"""Main loop."""
port = options.port
if len(sys.argv) > 1:
port = int(sys.argv[1])
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(False)
sock.bind(('', port))
sock.listen(5)
io_loop = tornado.ioloop.IOLoop.current()
callback = functools.partial(connection_ready, sock)
io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
print 'Tornado Proxy working on port: %s' % port
io_loop.start()
But when I try to connect to my proxy (for example to add filters: http://127.0.0.1:8000/admin/filters/ - I have an handler written to handle this url)
I get specyfic errors:
ERROR:tornado.application:Exception in callback (3, )
Traceback (most recent call last):
File "/home/dave/.virtualenvs/teleV1/local/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "/home/dave/.virtualenvs/teleV1/local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "manage.py", line 35, in connection_ready
connection.send(server)
TypeError: send() argument 1 must be convertible to a buffer, not HTTPServer
I understand that I cannot send a HTTPServer through the connection (from one end to the other), it must be a buffer.
My first idea was to send the buffor from the handler that handles a URL (for example class ProxyHandler(tornado.web.RequestHandler)),
but how can I handle diffrent urls and diffrent handlers?
After a few approaches I changed my Proxy app written as a Tornado app to a pure Python code, that handles responses from remote addresses and does the filter stuff. I think this is the best and fastest thing that I could do.

Multithreaded web server in python

I'm trying to create multithreaded web server in python, but it only responds to one request at a time and I can't figure out why. Can you help me, please?
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
from SocketServer import ThreadingMixIn
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
from time import sleep
class ThreadingServer(ThreadingMixIn, HTTPServer):
pass
class RequestHandler(SimpleHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-type', 'text/plain')
sleep(5)
response = 'Slept for 5 seconds..'
self.send_header('Content-length', len(response))
self.end_headers()
self.wfile.write(response)
ThreadingServer(('', 8000), RequestHandler).serve_forever()
Check this post from Doug Hellmann's blog.
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from SocketServer import ThreadingMixIn
import threading
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.end_headers()
message = threading.currentThread().getName()
self.wfile.write(message)
self.wfile.write('\n')
return
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == '__main__':
server = ThreadedHTTPServer(('localhost', 8080), Handler)
print 'Starting server, use <Ctrl-C> to stop'
server.serve_forever()
I have developed a PIP Utility called ComplexHTTPServer that is a multi-threaded version of SimpleHTTPServer.
To install it, all you need to do is:
pip install ComplexHTTPServer
Using it is as simple as:
python -m ComplexHTTPServer [PORT]
(By default, the port is 8000.)
In python3, you can use the code below (https or http):
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn
import threading
USE_HTTPS = True
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write(b'Hello world\t' + threading.currentThread().getName().encode() + b'\t' + str(threading.active_count()).encode() + b'\n')
class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
pass
def run():
server = ThreadingSimpleServer(('0.0.0.0', 4444), Handler)
if USE_HTTPS:
import ssl
server.socket = ssl.wrap_socket(server.socket, keyfile='./key.pem', certfile='./cert.pem', server_side=True)
server.serve_forever()
if __name__ == '__main__':
run()
You will figure out this code will create a new thread to deal with every request.
Command below to generate self-sign certificate:
openssl req -x509 -newkey rsa:4096 -nodes -out cert.pem -keyout key.pem -days 365
If you are using Flask, this blog is great.
It's amazing how many votes these solutions that break streaming are getting. If streaming might be needed down the road, then ThreadingMixIn and gunicorn are no good because they just collect up the response and write it as a unit at the end (which actually does nothing if your stream is infinite).
Your basic approach of combining BaseHTTPServer with threads is fine. But the default BaseHTTPServer settings re-bind a new socket on every listener, which won't work in Linux if all the listeners are on the same port. Change those settings before the serve_forever() call. (Just like you have to set self.daemon = True on a thread to stop ctrl-C from being disabled.)
The following example launches 100 handler threads on the same port, with each handler started through BaseHTTPServer.
import time, threading, socket, SocketServer, BaseHTTPServer
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
if self.path != '/':
self.send_error(404, "Object not found")
return
self.send_response(200)
self.send_header('Content-type', 'text/html; charset=utf-8')
self.end_headers()
# serve up an infinite stream
i = 0
while True:
self.wfile.write("%i " % i)
time.sleep(0.1)
i += 1
# Create ONE socket.
addr = ('', 8000)
sock = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(addr)
sock.listen(5)
# Launch 100 listener threads.
class Thread(threading.Thread):
def __init__(self, i):
threading.Thread.__init__(self)
self.i = i
self.daemon = True
self.start()
def run(self):
httpd = BaseHTTPServer.HTTPServer(addr, Handler, False)
# Prevent the HTTP server from re-binding every handler.
# https://stackoverflow.com/questions/46210672/
httpd.socket = sock
httpd.server_bind = self.server_close = lambda self: None
httpd.serve_forever()
[Thread(i) for i in range(100)]
time.sleep(9e9)
A multithreaded https server in python3.7
from http.server import BaseHTTPRequestHandler, HTTPServer
from socketserver import ThreadingMixIn
import threading
import ssl
hostName = "localhost"
serverPort = 8080
class MyServer(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(bytes("<html><head><title>https://pythonbasics.org</title></head>", "utf-8"))
self.wfile.write(bytes("<p>Request: %s</p>" % self.path, "utf-8"))
self.wfile.write(bytes("<p>Thread: %s</p>" % threading.currentThread().getName(), "utf-8"))
self.wfile.write(bytes("<p>Thread Count: %s</p>" % threading.active_count(), "utf-8"))
self.wfile.write(bytes("<body>", "utf-8"))
self.wfile.write(bytes("<p>This is an example web server.</p>", "utf-8"))
self.wfile.write(bytes("</body></html>", "utf-8"))
class ThreadingSimpleServer(ThreadingMixIn,HTTPServer):
pass
if __name__ == "__main__":
webServer = ThreadingSimpleServer((hostName, serverPort), MyServer)
webServer.socket = ssl.wrap_socket(webServer.socket, keyfile='./privkey.pem',certfile='./certificate.pem', server_side=True)
print("Server started http://%s:%s" % (hostName, serverPort))
try:
webServer.serve_forever()
except KeyboardInterrupt:
pass
webServer.server_close()
print("Server stopped.")
you can test it in a browser: https://localhost:8080
the running result is:
enter image description here
enter image description here
remind that you can generate your own keyfile and certificate use
$openssl req -newkey rsa:2048 -keyout privkey.pem -x509 -days 36500 -out certificate.pem
To learn details about creating self-signed certificate with openssl:https://www.devdungeon.com/content/creating-self-signed-ssl-certificates-openssl

Categories

Resources