So I found the following sample code which allows for a basic python HTTP server to be established at a given url and port. I am quite inexperienced with web servers and am trying to create handlers for certain GET requests to this server. However, I cannot figure out how to actually create handlers for a GET request made by another computer when accessing this URL remotely. Any suggestions?
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
# just send back the same data, but upper-cased
self.request.sendall(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "url" , PORT
# Create the server, binding to localhost on port 9999
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()
Here is very simple example of how it could work. You would start this and call it with this, for example:
curl -i 'http://localhost:5001/foo/bar?foo=bar' -X POST -d '{"Foo":"Bar"}'
HTTP/1.1 200 OK
Some response%
It is missing tons of things, but this should at least give you some sort of idea.
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print self.data
self.parse_request(self.data)
func, args = self.path.split("/", 1)
args = args.split("/")
resp = getattr(self, func)(*args)
self.request.sendall("HTTP/1.1 200 OK\n")
self.request.sendall("\n")
self.request.sendall(resp)
def parse_request(self, req):
headers = {}
lines = req.splitlines()
inbody = False
body = ''
for line in lines[1:]:
if line.strip() == "":
inbody = True
if inbody:
body += line
else:
k, v = line.split(":", 1)
headers[k.strip()] = v.strip()
method, path, _ = lines[0].split()
self.path = path.lstrip("/")
self.method = method
self.headers = headers
self.body = body
self.path, self.query_string = self.path.split("?")
def foo(self, *args):
print self.path
print self.query_string
print self.body
print self.headers
print self.method
return "Some response"
if __name__ == "__main__":
server = SocketServer.TCPServer(("localhost", 5001), MyTCPHandler)
server.serve_forever()
Related
I need to test a device update function. The function opens a socket on a host and sends a block of text.
The update can take up to 120 seconds. It returns a code for success/failure. To allow continued functioning of the program the update is launched in a thread.
I cannot control the response of the device. The simulation needs to be able to hold an open connection for at least 120 seconds.
It does not need to be safe or scalable since it will only be used for an integration test. The simplest solution is preferred. Pure python is best, but a docker is also acceptable.
I wrote this up based on rdas's pointer.
import json
import logging
import socket
import socketserver
import threading
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
class LongRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
# Echo the back to the client
data = json.loads(self.request.recv(1024).decode())
t = 0
while t < data['delay']:
time.sleep(1)
print(".", end='')
t += 1
if t % 80 == 0:
print("\n")
print("\n")
self.request.send(b"ok")
class Server():
def __init__(self, host='localhost', port=0):
self.host = host
self.port = port
self.ip = None
self.server = None
def run(self):
address = (self.host, self.port) # let the kernel assign port if port=0
self.server = socketserver.TCPServer(address, LongRequestHandler)
self.ip, self.port = self.server.server_address # what port was assigned?
t = threading.Thread(target=self.server.serve_forever)
t.setDaemon(True) # don't hang on exit
t.start()
return True
def send_request(self, data: dict ):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.ip, self.port))
message = json.dumps(data).encode()
s.send(message)
response = s.recv(1024)
s.close()
return response
def __exit__(self):
self.server.shutdown()
self.server.socket.close()
if __name__ == '__main__':
# For simple testing and config example...
server = Server()
server.run()
# Send the data
d = dict(delay=5) # set delay here to desired
out = server.send_request(d)
print('Received: {!r}'.format(out))
Trying to read headers in twisted middle proxy server.
Here you can see simple twisted server (stolen from stackoverflow, yeah...). It nicely works, but I need to switch proxies based on request headers. Can't understand where I can get headers here. It workd with hard coded proxie server but the idea is to switch proxies based on requests.
Any ideas please? Thanks for your time.
Here is the code:
#!/usr/bin/env python
LISTEN_PORT = 8080
SERVER_PORT = 3128
SERVER_ADDR = "89.40.127.96"
from twisted.internet import protocol, reactor
class ServerProtocol(protocol.Protocol):
def __init__(self):
self.buffer = None
self.client = None
def connectionMade(self):
factory = protocol.ClientFactory()
factory.protocol = ClientProtocol
factory.server = self
reactor.connectTCP(SERVER_ADDR, SERVER_PORT, factory)
# Client => Proxy
def dataReceived(self, data):
if self.client:
self.client.write(data)
else:
self.buffer = data
# Proxy => Client
def write(self, data):
self.transport.write(data)
class ClientProtocol(protocol.Protocol):
def connectionMade(self):
self.factory.server.client = self
self.write(self.factory.server.buffer)
self.factory.server.buffer = ''
# Server => Proxy
def dataReceived(self, data):
self.factory.server.write(data)
# Proxy => Server
def write(self, data):
if data:
self.transport.write(data)
def main():
factory = protocol.ServerFactory()
factory.protocol = ServerProtocol
reactor.listenTCP(LISTEN_PORT, factory)
print "server"
reactor.run()
if __name__ == '__main__':
main()
I'm a newbie to creating manually sockets. My OS is ubuntu. I've got an proxy server written python using Tornado, everything is fine when I use the "fast version" starting the app, I mean the:
if __name__ == "__main__":
app = make_app()
port = options.port # default 8000
if len(sys.argv) > 1:
port = int(sys.argv[1])
app.listen(port)
print 'tornado working on port %s' % port
tornado.ioloop.IOLoop.current().start()
But when i want to change it to use the 'socket version' it seems that I'm doing something wrong. I get an error saying that the address is already used.
code:
def make_app():
return MyApplication()
def connection_ready(sock, fd, events):
while True:
try:
connection, address = sock.accept()
except socket.error as e:
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
raise
return
connection.setblocking(0)
app = make_app()
app.listen(8000) # I get here an error: [Errno 98] Address already in use
if __name__ == "__main__":
port = options.port # default port 8000
if len(sys.argv) > 1:
port = int(sys.argv[1])
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(False)
sock.bind(("", port))
sock.listen(128)
io_loop = tornado.ioloop.IOLoop.current()
callback = functools.partial(connection_ready, sock)
io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
io_loop.start()
I'm trying to implement the same way that the documentation says (http://www.tornadoweb.org/en/stable/ioloop.html) but I don't see it starting the app in there.
Could someone tell me what is the proper way to start an app using sockets? I'm trying to accomplish an application that is available when the sever accepts the incoming socket. (So every client that connects to my listining port described in the main function at lines:sock.bind(("", port)) and sock.listen(128) will get a new socket and have access to the application).
Edit: I'm adding my proxy class:
class ProxyHandler(tornado.web.RequestHandler):
SUPPORTED_METHODS = ['GET', 'POST']
def data_received(self, chunk):
pass
def compute_etag(self):
return None # disable tornado Etag
def handle_response(self, response):
if response.error and not isinstance(response.error, tornado.httpclient.HTTPError):
self.set_status(500)
self.write('Internal server error:\n' + str(response.error))
else:
self.set_status(response.code, response.reason)
self._headers = tornado.httputil.HTTPHeaders() # clear tornado default header
for header, v in response.headers.get_all():
if header not in ('Content-Length', 'Transfer-Encoding', 'Content-Encoding', 'Connection'):
self.add_header(header, v) # some header appear multiple times, eg 'Set-Cookie'
secured_page = False
for page in secure_pages:
if page in self.request.uri:
secured_page = True
self.set_header('Content-Length', len(response.body))
self.write(response.body)
break
if response.body and not secured_page:
c.execute('SELECT filter_name FROM filters WHERE filter_type=1')
tags = c.fetchall()
soup = BeautifulSoup(response.body, 'html.parser')
for row in tags:
catched_tags = soup.find_all(str(row[0]))
if catched_tags:
print 'catched: %s of <%s> tags' % (len(catched_tags), str(row[0]))
for tag in catched_tags:
tag.extract()
new_body = str(soup)
self.set_header('Content-Length', len(new_body))
self.write(new_body)
self.finish()
#tornado.web.asynchronous
def get(self):
logger.debug('Handle %s request to %s', self.request.method, self.request.uri)
body = self.request.body
if not body:
body = None
try:
if 'Proxy-Connection' in self.request.headers:
del self.request.headers['Proxy-Connection']
c.execute('SELECT filter_name FROM filters WHERE filter_type=2')
urls = c.fetchall()
for url in urls:
if url[0] in self.request.path:
self.set_status(403)
self.finish()
return
fetch_request(self.request.uri, self.handle_response,
method=self.request.method, body=body, headers=self.request.headers, follow_redirects=False,
allow_nonstandard_methods=True)
except tornado.httpclient.HTTPError as e:
if hasattr(e, 'response') and e.response:
self.handle_response(e.response)
else:
self.set_status(500)
self.write('Internal server error:\n' + str(e))
self.finish()
#tornado.web.asynchronous
def post(self):
return self.get()
And my urls for the application:
urls = [
url(r"/admin/$", mainHandlers.MainHandler),
url(r"/admin/delete_filter/", mainHandlers.DataDeleteHandler),
url(r"/admin/filters/$", mainHandlers.DataGetter),
url(r"/admin/new_filter/$", mainHandlers.FormHandler),
url(r"/admin/stats/$", mainHandlers.StatsTableHandler),
url(r"/admin/stats/query/$", mainHandlers.AjaxStatsGetHandler),
url(r"/static/", StaticFileHandler, dict(path=settings['static_path'])),
url(r'.*', myProxy.ProxyHandler),
]
It says the port is already in use because it is. You're listening on port 8000 at least twice: once in the __main__ block when you call sock.listen, and again in the connection_ready handler when you call app.listen() (which creates another socket and tries to bind it to port 8000). You need to remove the app.listen() line, but I don't understand what you're trying to do well enough to say what you should do instead.
If you start app on Windows, you must wait for the firewall unblock. In windows it is safe to assume that if an application occupies a port it is blocked for use by other processes that might listen to packets not intended for them.
I've rewitten my Proxy to pure Python code on sockets, I'm not using URL's now and I only handle the responses from the remote addresses. I'm not using any framework
I'm working on a very simple server. This server should be able to do some predefined commands requested by clients.
I would like to store information about connected clients separately. For example, I want server to add a number of the particular clients requests.
Here is an example:
SERVER
CLIENT1
CLIENT2
CLIENT1> print 'stuff'
SERVER>> REQUESTS: 1 OUTPUT: stuff
CLIENT2> print 'simple sentence'
SERVER>> REQUESTS: 1 OUTPUT: simple sentence
CLIENT> print 'hilarious'
SERVER>> REQUESTS: 2 OUTPUT: hilarious
My code is simple:
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(1024)
command = data.split(' ')[0]
arguments = data.split(' ')[1:]
cur_thread = threading.current_thread()
output = do_command(command,arguments)
response = "{}: {}".format(cur_thread.name, output)
self.request.sendall(response)
if __name__ == "__main__":
commands.register_commands()
HOST, PORT = _host, int(_port)
server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)
ip, port = server.server_address
# start server
print "Running on: %s:%s" % (HOST, PORT)
server.serve_forever()
So the thing I want to know is how could I store information about each client. I was considering to create a class Client and make an object after each connection but I don't know where should I create this objects.
You could use the client_address property of the handler to identify clients, and track requests in a global dict:
from collections import defaultdict
client_requests = defaultdict(int)
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
...
client_requests[self.client_address[0]] += 1
response = "{} ({}): {}".format(cur_thread.name, client_requests[self.client_address[0]], output)
Note that since you're using a threaded server, you'll probably need to add some locking code to protect the writes to client_requests, but that's an exercise left to the reader.
When a client (ie. web browser) points to localhost:8080, its request should be sent to an address defined by SERVER_ADDR such as http://www.yahoo.com. It's more like a router or load balancer, than a proxy.
Problem: When my web browser points to localhost:8080, nothing is returned to it. On the Python console, the HTTP request header can be seen to have reached the Twisted app. Maybe this cannot be used for redirecting the browser's request?
from twisted.internet import protocol, reactor
LISTEN_PORT = 8080
SERVER_PORT = 80
SERVER_ADDR = 'http://www.yahoo.com'
class ServerProtocol(protocol.Protocol):
def __init__(self):
self.buffer = None
self.client = None
def connectionMade(self):
factory = protocol.ClientFactory()
factory.protocol = ClientProtocol
factory.server = self
reactor.connectTCP(SERVER_ADDR, SERVER_PORT, factory)
# Client => Proxy
def dataReceived(self, data):
print 'Data received from Client:'
if self.client:
self.client.write(data)
else:
data = data.replace('localhost:8080', SERVER_ADDR)
print data
self.buffer = data
# Proxy => Client
def write(self, data):
self.transport.write(data)
class ClientProtocol(protocol.Protocol):
def connectionMade(self):
self.factory.server.client = self
self.write(self.factory.server.buffer)
self.factory.server.buffer = ''
# Server => Proxy
def dataReceived(self, data):
print 'Data received from Server: '
print data
self.factory.server.write(data)
# Proxy => Server
def write(self, data):
if data:
self.transport.write(data)
def main():
factory = protocol.ServerFactory()
factory.protocol = ServerProtocol
reactor.listenTCP(LISTEN_PORT, factory)
reactor.run()
if __name__ == '__main__':
main()
It partially works for me: I get an HTTP error 400 bad request from the Yahoo server in my browser when I visit http://localhost:8080. This is because you're replacing the Host: localhost:8080 section of the GET request with Host: http://www.yahoo.com and the protocol here is invalid. It should just be Host: www.yahoo.com.
That said, it then responds with a HTTP/1.1 301 Moved Permanently redirect and it you run into other problems.
Edit: you should take a look at the proxy example on the twisted website (https://twistedmatrix.com/documents/12.3.0/web/examples/#auto2) and dive into the source to see how it's implemented.
from twisted.web import proxy, http
from twisted.internet import reactor
class ProxyFactory(http.HTTPFactory):
def buildProtocol(self, addr):
return proxy.Proxy()
reactor.listenTCP(8080, ProxyFactory())
reactor.run()