Develop Reference

intercepting https traffic from the browser python

So I'm trying to create a web proxy to intercept the browser traffic, however working with http was easy and I was able to do easily but when it comes to https I get this not informative error and I don't know what is wrong, all I can find is that the exception is triggered when this line is executed (client_socket, address) = ssl_socket.accept()
import socket
import thread
import ssl
def proxy_server():
server_scoket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# http_listener(server_scoket)
https_listener(server_scoket)
# server_scoket.close()
def https_listener(server_scoket):
ssl_socket = ssl.wrap_socket(server_scoket,keyfile='localhost.key',certfile='localhost.crt',server_side=1)
ssl_socket.bind(('127.0.0.1',9000))
ssl_socket.listen(50)
while True:
try :
(client_socket, address) = ssl_socket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except Exception as e:
print "Error {}".format(e)
to test the code I tried to browse to the python website : www.python.org and here is the output :
Error [SSL: HTTPS_PROXY_REQUEST] https proxy request (_ssl.c:727)
UPDATE 1 :
the full code :
import socket
import thread
import ssl
def proxy_server():
server_scoket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
http_listener(server_scoket)
# https_listener(server_scoket)
# server_scoket.close()
def https_listener(server_scoket):
context = ssl.SSLContext(ssl.PROTOCOL_TLS)
context.load_cert_chain('localhost.crt', 'localhost.key')
server_scoket.bind(('127.0.0.1',9000))
server_scoket.listen(5)
ssl_socket = context.wrap_socket(server_scoket,server_side=True)
while True:
try :
(client_socket, address) = ssl_socket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except Exception as e:
print "Error {}".format(e)
def http_listener(server_scoket):
try :
server_scoket.bind(('127.0.0.1',9000))
server_scoket.listen(50)
while True:
(client_socket, address) = server_scoket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except KeyboardInterrupt :
print "\n shutting down"
except Exception as e :
print "Error : {}".format(e)
def proxy_thread(connection, client_address):
request = connection.recv(5000)
url = request.split('\n')[0].split(' ')[1]
splited_request = request.split('\r\n\r\n')
headers = splited_request[0].split('\n')
body = splited_request[1]
port = 0
print "---------"
print request
print "---------"
splitted_url = url.split(':')
url = splitted_url[1][2:]
base_url = url.split('/')[0]
path_url = url[len(base_url):]
if (len(splitted_url) < 3):
port = 80
else:
port = splitted_url[2]
try :
splited_line = headers[0].split(' ')
if (splited_line[0] != "CONNECT"):
headers[0] = "{} {} {}".format(splited_line[0],path_url,splited_line[2])
else:
base_url = headers[0].split(' ')[1]
new_headers = ""
for index,header in enumerate(headers) :
if (index != len(headers) - 1):
headers[index] = "{}\n".format(header)
new_headers = "".join(headers)
request = "{} \r\n\r\n {}".format(new_headers,body)
if (splitted_url[0] == "https"):
https_proxy(base_url,443,request,connection)
else:
http_proxy(base_url,443,request,connection)
connection.close()
except OSError, message:
if s:
s.close()
if connection:
connection.clos()
print "Error messgae : "+ message
except Exception as e:
print "Error : {}".format(e)
def http_proxy(base_url,port,request,connection):
print "------ http request start -----"
print request
print "------ request ends -----"
print port
if(request.split(' ')[0] == 'CONNECT'):
reply = "HTTP/1.1 200 OK\r\n\r\n"
connection.sendall(reply)
print request.split(' ')[0]
print "replied HTTP/1.1 200 OK\r\n\r\n"
return
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((base_url,port))
s.sendall(request)
while True:
data = s.recv(50000)
if (len(data) > 0):
connection.sendall(data)
else:
break
s.close()
def https_proxy(base_url,port,request,connection):
print "------ https request start -----"
print request
print "------ request ends -----"
print port
sp_base_url = base_url.split(':')
base_url = sp_base_url[0] if len(sp_base_url) > 1 else base_url
context = ssl.create_default_context()
sock = socket.create_connection(("cdn.sstatic.net",443))
print port
context.load_verify_locations('/etc/ssl/certs/ca-certificates.crt')
ssock = context.wrap_socket(sock, server_hostname="cdn.sstatic.net")
print ssock.version()
ssock.sendall(request)
while True:
data = ssock.recv(5000)
if (len(data) > 0):
print data
connection.sendall(data)
else:
break
sock.close()
connection.close()
print "You can use the following proxy : \n host : 127.0.0.1 \n port : 9000 \n"
proxy_server()
the execution output :
You can use the following proxy :
host : 127.0.0.1
port : 9000
---------
CONNECT www.pythonconverter.com:443 HTTP/1.1
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0
Proxy-Connection: keep-alive
Connection: keep-alive
Host: www.pythonconverter.com:443
---------
------ http request start -----
CONNECT www.pythonconverter.com:443 HTTP/1.1
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0
Proxy-Connection: keep-alive
Connection: keep-alive
Host: www.pythonconverter.com:443
------ request ends -----
443
CONNECT
replied HTTP/1.1 200 OK
but then from my understanding I should get a new connection or at least a request on the same socket from the browser, however that does not happen !

Python socket programmed web server doesnt send information to browser

I create a simple web server using python socket programming. When I access it using a socket programmed client I get this response (which seems to be good):
HTTP/1.0 200 OK
Content-Length: 145
Content-Type: text/html
"""<!DOCTYPE html>
<html>
<body>
<h2>HTML Links</h2>
<p>Visit our HTML tutorial</p>
</body>
</html>"""
However, when I try to access 127.0.0.1:80 on the browser it says:
127.0.0.1 didn’t send any data. ERR_EMPTY_RESPONSE
Web Server Code:
import socket
import os
def get_content_type(filename):
index = filename.rfind('.')
extension = filename[index+1:len(filename)]
if(extension == 'txt' or extension == 'html'):
return 'Content-Type: text/html\n'
elif(extension == 'jpg'):
return 'Content Type: image/jpeg\n'
elif(extension == 'js'):
return 'Content Type: text/javascript; charset=UTF 8\n'
elif(extension == 'css'):
return 'Content Type: text/css\n'
pass
def check_client_request(client_request):
request_splitted = client_request.split()
if(len(request_splitted) != 3):
return False
if(request_splitted[0] != 'GET'):
return False
if(request_splitted[1].find('http://') != 0):
return False
if(request_splitted[1].count('/') < 3):
return False
if(request_splitted[2] != 'HTTP/1.1\\r\\n'):
return False
return True
def recieve_client_request(client_socket):
client_request = client_socket.recv(1024)
return client_request.decode('utf-8')
def handle_client_request(request):
try:
filename = request.split()[1].split('/')[3]
except:
return 'File not found'
if(filename == ''):
filename = 'index.html'
path = f'C:\\Users\\Eitan\\Desktop\\Python-Course\\SOCKETWEBSERVER\\{filename}'
print(path)
response = ''
if(os.path.isfile(path)):
try:
requested_file = open(path, 'r')
file_content = requested_file.read()
requested_file.close()
response = 'HTTP/1.0 200 OK\n'
content_length = len(file_content.encode('utf-8'))
response += f'Content-Length: {content_length}\n'
response += get_content_type(filename)
response += '\n'
response += f'"""{file_content}"""'
except:
response = 'HTTP/1.1 404 Not Found\n'
else:
response = 'HTTP/1.1 404 Not Found\n'
return response
def send_response(client_socket, response):
try:
client_socket.send(response.encode('utf-8'))
print('Response Sent')
except:
print('Couldnt send response.')
def main():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(('0.0.0.0', 80))
server_socket.listen(1)
while True:
client_socket = server_socket.accept()[0]
client_request = recieve_client_request(client_socket)
if(check_client_request(client_request)):
response = handle_client_request(client_request)
send_response(client_socket, response)
client_socket.close()
else:
client_socket.close()
if(__name__ == '__main__'):
main()
Client Code:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 80))
request = input("Command: ").encode('utf-8')
client_socket.send(request)
response = client_socket.recv(1024)
print(response.decode('utf-8'))
if(__name__ == '__main__'):
main()

if(request_splitted[1].find('http://') != 0):
return False
You expect the browser to send a request like this
GET http://domain/page HTTP/1.1
...
But, a normal HTTP request does not include protocol and host but only the page, i.e. it looks like this
GET /page HTTP/1.1
...
Since you treat the valid request from the browser as invalid you close the connection and thus no response is sent to the browser.
Note that HTTP is not that simple as it might look. There is an actual standard for this which is quite long and which you are expected to follow when implementing a HTTP server or client.

keep-alive messages not being sent when using python request module

I am observing that with python requests module, HTTP keep-alive is not being honored.
I dont see Acks for keep-alive being sent from the host where i am running the python script.
Please let me know how it can be fixed.Following is my code:
import json
import requests
import logging
import sys
import time
from threading import Thread
logging.basicConfig(level=logging.DEBUG)
class NSNitro:
def __init__(self,*args):
if len(args) > 2:
self.ip = args[0]
self.username = args[1]
self.password = args[2]
self.session_id = None
url = 'http://'+self.ip+'/nitro/v1/config/login'
payload = { "login": { "username":"nsroot", "password":"nsroot" }}
headers = {"Content-type": "application/json", 'Connection': 'keep-alive'}
try:
r = requests.post(url=url,headers=headers,data=json.dumps(payload),timeout=5)
logging.info(r.json()["sessionid"])
if(r.json()["sessionid"] != None):
self.session_id = r.json()["sessionid"]
except requests.exceptions.RequestException:
logging.critical("Some error occurred during connection")
else:
logging.error("Not sufficient parameters provided.Required : ipaddress , username , password")
def install_build(self,build_url):
url = 'http://ip/nitro/v1/config/install'
headers = {"Content-type": "application/json","Connection": "keep-alive"}
payload = {"install": {"url": build_url}}
try:
cookie = {"NITRO_AUTH_TOKEN": self.session_id}
r = requests.post(timeout=5, url=url, data=json.dumps(payload), headers=headers,cookies=cookie)
except requests.exceptions.RequestException:
print("Connection Error occurred")
raise '''this will give details of exception'''
else:
assert r.status_code == 201, "Status code seen: " + str(r.status_code) + "\n" + "Error message from system: " + \
r.json()["message"]
print("Successfully triggered job on device to install build")
def __del__(self):
logging.debug("Deleted the object")
if __name__ == '__main__':
ns_session = NSNitro(ip,username,password)
url_i = 'https://myupload-server.net/build-13.0-480.16.tgz'
t1 = Thread(target=ns_session.install_build,args=(url_i,))
t1.start()
''' while t1.is_alive():
t2 = Thread(target=ns_session.get_installed_version,)
t2.start()
t2.join()'''
time.sleep(100)
logging.info("Install thread completed")
t1.join()
ns_session.logout()
When the request is posted using curl command, the acks are sent in specified keep-alive intervals. Without ack being sent , server is resetting the connection.

How to get the IP address of a client on HTTP server based python?

I built a simple HTTP server based python, and I would like to print on the console the IP of every client ( in this line - print("IP address of the client "+IP)). Can you help me please do so?
I attached the full code of the server, Thanks!
The server is an HTTP server based Python, I used select and socket.
The reason for the need of the client IP address is to create a dictionary of IP of users. Thanks!
import select, socket, queue, os
DEFAULT_URL = r'/signup.html'
ERROR_404_URL = r'/errors/404.html'
ROOT_PATH = r'/web'
REDIRECTION_LIST = [r"/index.html", r"/index.htm", r"index.html"]
IP = "0.0.0.0"
PORT = 12345
IMAGE_TYPES = ["png", "jpg", "bmp", "gif", "jpeg"]
SERVER_STATUS = {"OK": 200, "Redirect": 302, "Not Found": 404}
BUFFER_LENGTH = 1024
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# server.setblocking(0)
server.bind((IP, PORT))
server.listen(5)
inputs = [server]
outputs = [] #requests
message_queues = {}
users ={}
def get_file_data(file_path):
""" Get and return data from file in :param file_path"""
try:
file_handler = open(file_path, 'rb')
# read file content
response_content = file_handler.read()
file_handler.close()
return response_content
except Exception:
print ("Warning, file not found. Serving response code 404\n")
print("file path "+file_path)
print("Done")
return None
def get_file_info(client_request):
""" Get absolute response file path and response file type by parsing :param client_request"""
str_array = client_request.split(" ")
file_path_parameters = str_array[1]
strings = file_path_parameters.split("?")
if '?' in file_path_parameters:
get_req = strings[1]
get_request = get_req.split("=")
print("get_request "+get_request[0])
print("ip of the client "+IP) # HERE it should print the
#client IP
# print("string "+ strings[1])
file_path =strings[0]
print("file path " + file_path)
if file_path == r"/":
file_path = DEFAULT_URL
print("file path "+ file_path)
file_type = file_path.split(".")[1]
abs_file_path = ROOT_PATH + file_path # "/test.html"
print(abs_file_path)
return abs_file_path, file_type
def header(url, file_type):
######################################
# headers
######################################
headers = ""
http_version = "HTTP/1.1 200 OK"
content_length = str(os.path.getsize(url))
content_type = file_type
if content_type == "html":
headers = 'HTTP/1.1 200 OK' + '\nContent-Type: text/html; charset=utf-8 \nContent-Length: ' + str(content_length) + '\n\n'
elif content_type == "css":
headers = 'HTTP/1.1 200 OK\nContent-Type: text/css \nContent-Length: ' + str(content_length) + '\n\n'
elif content_type == "js":
headers = 'HTTP/1.1 200 OK' +'\nContent-Type: text/javascript; charset=utf-8 \nContent-Length: ' + str(content_length) + '\n\n'
elif file_type in IMAGE_TYPES:
headers = 'HTTP/1.1 200 OK\nContent-Type: image/xyz \nContent-Length: ' + content_length + '\n\n'
else:
headers = 'HTTP/1.1 200 OK\nContent-Type: ' + content_type + '\nContent-Length: ' + str(content_length) + '\n\n'
return headers.encode()
# further headers
# current_date = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
# response_header += 'Date: ' + current_date + '\n'
# Important: "\n" twice - because html page body
# starts with first empty line
# response_header += 'Server: Allam-HTTP-Server\n\n'
# signal that the connection will be closed
# after completing the request
response_header += 'Connection: close\n\n'
print("response_header = ", response_header)
return response_header.encode() + file_data
def main():
""" Main loop for connect, read and write to/from sockets"""
while inputs:
readable, writable, exceptional = select.select(
inputs, outputs, inputs+outputs, 1)
for s in readable:
if s is server: #אם זה של הסרבר
new_client_socket, client_address = s.accept()
print("accepted")
# new_client_socket.setblocking(0)
inputs.append(new_client_socket)
message_queues[new_client_socket] = queue.Queue()
else:
data = s.recv(1024)
if data: #המחרוזת לא ריקה
message_queues[s].put(data)
if s not in outputs:
outputs.append(s)
else: # אם זה ריק כלומר להתנתק צריך להשמיד
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del message_queues[s]
for s in writable:
try:
next_msg = message_queues[s].get_nowait()
except queue.Empty:
outputs.remove(s)
else:
file_path, file_type = get_file_info(next_msg.decode())
file_data = get_file_data(file_path)
# file_size = len(file_data) # another way: num_of_bytes = os.stat(file_path).st_size
http_response = header(file_path, file_type)+file_data
s.send(http_response)
for s in exceptional:
inputs.remove(s)
if s in outputs:
outputs.remove(s)
s.close()
del message_queues[s]
if __name__ == '__main__':
main()

You get the client address from s.accept() -- it's a tuple of IP address (str) and port (int). Your code does not use this variable at all and cares only about the socket.
new_client_socket, client_address = s.accept()
You only pass the client request string to get_file_info, so it doesn't know anything about the client it is currently serving. Save the client address somewhere, maybe in a dict mapping sockets to such tuples?
More information: https://docs.python.org/3/library/socket.html#socket.socket.accept

How to force http.client to send chunked-encoding HTTP body in python?

I want to send chunked HTTP body to test my own HTTP server.
So I wrote this python code:
import http.client
body = 'Hello World!' * 80
conn = http.client.HTTPConnection("some.domain.com")
url = "/some_path?arg=true_arg"
conn.request("POST", url, body, {"Transfer-Encoding":"chunked"})
resp = conn.getresponse()
print(resp.status, resp.reason)
I expect the HTTP request's body is transferrd chunked,
but I capture the network package with Wireshark, the HTTP request's body is not transferred chunked.
How to transfer chunked body by http.client lib in python?

OK, I get it.
First, write my own chunked encode function.
Then use putrequest(), putheader(), endheaders() and send() instead of request()
import http.client
def chunk_data(data, chunk_size):
dl = len(data)
ret = ""
for i in range(dl // chunk_size):
ret += "%s\r\n" % (hex(chunk_size)[2:])
ret += "%s\r\n\r\n" % (data[i * chunk_size : (i + 1) * chunk_size])
if len(data) % chunk_size != 0:
ret += "%s\r\n" % (hex(len(data) % chunk_size)[2:])
ret += "%s\r\n" % (data[-(len(data) % chunk_size):])
ret += "0\r\n\r\n"
return ret
conn = http.client.HTTPConnection(host)
url = "/some_path"
conn.putrequest('POST', url)
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
conn.send(chunk_data(body, size_per_chunk).encode('utf-8'))
resp = conn.getresponse()
print(resp.status, resp.reason)
conn.close()

I'd suggest that if you already know the size of your data like in the answer given you could just set the Content-Length and send it all back in one hit, which is kind of what you're doing with the single call to conn.send anyway.
Chunked transfer encoding is most useful when you don't know how big the data is e.g. dynamically generated content. I've modified your code to illustrate:
import httplib
def write_chunk(conn, data):
conn.send("%s\r\n" % hex(len(data))[2:])
conn.send("%s\r\n" % data)
def dynamically_generate_data():
for i in range(80):
yield "hello world"
conn = httplib.HTTPConnection("localhost")
url = "/some_path"
conn.putrequest('POST', url)
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
for new_chunk in dynamically_generate_data():
write_chunk(conn, new_chunk)
conn.send('0\r\n')
resp = conn.getresponse()
print(resp.status, resp.reason)
conn.close()