Develop Reference

intercepting https traffic from the browser python

So I'm trying to create a web proxy to intercept the browser traffic, however working with http was easy and I was able to do easily but when it comes to https I get this not informative error and I don't know what is wrong, all I can find is that the exception is triggered when this line is executed (client_socket, address) = ssl_socket.accept()
import socket
import thread
import ssl
def proxy_server():
server_scoket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# http_listener(server_scoket)
https_listener(server_scoket)
# server_scoket.close()
def https_listener(server_scoket):
ssl_socket = ssl.wrap_socket(server_scoket,keyfile='localhost.key',certfile='localhost.crt',server_side=1)
ssl_socket.bind(('127.0.0.1',9000))
ssl_socket.listen(50)
while True:
try :
(client_socket, address) = ssl_socket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except Exception as e:
print "Error {}".format(e)
to test the code I tried to browse to the python website : www.python.org and here is the output :
Error [SSL: HTTPS_PROXY_REQUEST] https proxy request (_ssl.c:727)
UPDATE 1 :
the full code :
import socket
import thread
import ssl
def proxy_server():
server_scoket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
http_listener(server_scoket)
# https_listener(server_scoket)
# server_scoket.close()
def https_listener(server_scoket):
context = ssl.SSLContext(ssl.PROTOCOL_TLS)
context.load_cert_chain('localhost.crt', 'localhost.key')
server_scoket.bind(('127.0.0.1',9000))
server_scoket.listen(5)
ssl_socket = context.wrap_socket(server_scoket,server_side=True)
while True:
try :
(client_socket, address) = ssl_socket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except Exception as e:
print "Error {}".format(e)
def http_listener(server_scoket):
try :
server_scoket.bind(('127.0.0.1',9000))
server_scoket.listen(50)
while True:
(client_socket, address) = server_scoket.accept()
thread.start_new_thread(proxy_thread, (client_socket, address))
except KeyboardInterrupt :
print "\n shutting down"
except Exception as e :
print "Error : {}".format(e)
def proxy_thread(connection, client_address):
request = connection.recv(5000)
url = request.split('\n')[0].split(' ')[1]
splited_request = request.split('\r\n\r\n')
headers = splited_request[0].split('\n')
body = splited_request[1]
port = 0
print "---------"
print request
print "---------"
splitted_url = url.split(':')
url = splitted_url[1][2:]
base_url = url.split('/')[0]
path_url = url[len(base_url):]
if (len(splitted_url) < 3):
port = 80
else:
port = splitted_url[2]
try :
splited_line = headers[0].split(' ')
if (splited_line[0] != "CONNECT"):
headers[0] = "{} {} {}".format(splited_line[0],path_url,splited_line[2])
else:
base_url = headers[0].split(' ')[1]
new_headers = ""
for index,header in enumerate(headers) :
if (index != len(headers) - 1):
headers[index] = "{}\n".format(header)
new_headers = "".join(headers)
request = "{} \r\n\r\n {}".format(new_headers,body)
if (splitted_url[0] == "https"):
https_proxy(base_url,443,request,connection)
else:
http_proxy(base_url,443,request,connection)
connection.close()
except OSError, message:
if s:
s.close()
if connection:
connection.clos()
print "Error messgae : "+ message
except Exception as e:
print "Error : {}".format(e)
def http_proxy(base_url,port,request,connection):
print "------ http request start -----"
print request
print "------ request ends -----"
print port
if(request.split(' ')[0] == 'CONNECT'):
reply = "HTTP/1.1 200 OK\r\n\r\n"
connection.sendall(reply)
print request.split(' ')[0]
print "replied HTTP/1.1 200 OK\r\n\r\n"
return
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((base_url,port))
s.sendall(request)
while True:
data = s.recv(50000)
if (len(data) > 0):
connection.sendall(data)
else:
break
s.close()
def https_proxy(base_url,port,request,connection):
print "------ https request start -----"
print request
print "------ request ends -----"
print port
sp_base_url = base_url.split(':')
base_url = sp_base_url[0] if len(sp_base_url) > 1 else base_url
context = ssl.create_default_context()
sock = socket.create_connection(("cdn.sstatic.net",443))
print port
context.load_verify_locations('/etc/ssl/certs/ca-certificates.crt')
ssock = context.wrap_socket(sock, server_hostname="cdn.sstatic.net")
print ssock.version()
ssock.sendall(request)
while True:
data = ssock.recv(5000)
if (len(data) > 0):
print data
connection.sendall(data)
else:
break
sock.close()
connection.close()
print "You can use the following proxy : \n host : 127.0.0.1 \n port : 9000 \n"
proxy_server()
the execution output :
You can use the following proxy :
host : 127.0.0.1
port : 9000
---------
CONNECT www.pythonconverter.com:443 HTTP/1.1
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0
Proxy-Connection: keep-alive
Connection: keep-alive
Host: www.pythonconverter.com:443
---------
------ http request start -----
CONNECT www.pythonconverter.com:443 HTTP/1.1
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0
Proxy-Connection: keep-alive
Connection: keep-alive
Host: www.pythonconverter.com:443
------ request ends -----
443
CONNECT
replied HTTP/1.1 200 OK
but then from my understanding I should get a new connection or at least a request on the same socket from the browser, however that does not happen !

Python socket programmed web server doesnt send information to browser

I create a simple web server using python socket programming. When I access it using a socket programmed client I get this response (which seems to be good):
HTTP/1.0 200 OK
Content-Length: 145
Content-Type: text/html
"""<!DOCTYPE html>
<html>
<body>
<h2>HTML Links</h2>
<p>Visit our HTML tutorial</p>
</body>
</html>"""
However, when I try to access 127.0.0.1:80 on the browser it says:
127.0.0.1 didn’t send any data. ERR_EMPTY_RESPONSE
Web Server Code:
import socket
import os
def get_content_type(filename):
index = filename.rfind('.')
extension = filename[index+1:len(filename)]
if(extension == 'txt' or extension == 'html'):
return 'Content-Type: text/html\n'
elif(extension == 'jpg'):
return 'Content Type: image/jpeg\n'
elif(extension == 'js'):
return 'Content Type: text/javascript; charset=UTF 8\n'
elif(extension == 'css'):
return 'Content Type: text/css\n'
pass
def check_client_request(client_request):
request_splitted = client_request.split()
if(len(request_splitted) != 3):
return False
if(request_splitted[0] != 'GET'):
return False
if(request_splitted[1].find('http://') != 0):
return False
if(request_splitted[1].count('/') < 3):
return False
if(request_splitted[2] != 'HTTP/1.1\\r\\n'):
return False
return True
def recieve_client_request(client_socket):
client_request = client_socket.recv(1024)
return client_request.decode('utf-8')
def handle_client_request(request):
try:
filename = request.split()[1].split('/')[3]
except:
return 'File not found'
if(filename == ''):
filename = 'index.html'
path = f'C:\\Users\\Eitan\\Desktop\\Python-Course\\SOCKETWEBSERVER\\{filename}'
print(path)
response = ''
if(os.path.isfile(path)):
try:
requested_file = open(path, 'r')
file_content = requested_file.read()
requested_file.close()
response = 'HTTP/1.0 200 OK\n'
content_length = len(file_content.encode('utf-8'))
response += f'Content-Length: {content_length}\n'
response += get_content_type(filename)
response += '\n'
response += f'"""{file_content}"""'
except:
response = 'HTTP/1.1 404 Not Found\n'
else:
response = 'HTTP/1.1 404 Not Found\n'
return response
def send_response(client_socket, response):
try:
client_socket.send(response.encode('utf-8'))
print('Response Sent')
except:
print('Couldnt send response.')
def main():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(('0.0.0.0', 80))
server_socket.listen(1)
while True:
client_socket = server_socket.accept()[0]
client_request = recieve_client_request(client_socket)
if(check_client_request(client_request)):
response = handle_client_request(client_request)
send_response(client_socket, response)
client_socket.close()
else:
client_socket.close()
if(__name__ == '__main__'):
main()
Client Code:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 80))
request = input("Command: ").encode('utf-8')
client_socket.send(request)
response = client_socket.recv(1024)
print(response.decode('utf-8'))
if(__name__ == '__main__'):
main()

if(request_splitted[1].find('http://') != 0):
return False
You expect the browser to send a request like this
GET http://domain/page HTTP/1.1
...
But, a normal HTTP request does not include protocol and host but only the page, i.e. it looks like this
GET /page HTTP/1.1
...
Since you treat the valid request from the browser as invalid you close the connection and thus no response is sent to the browser.
Note that HTTP is not that simple as it might look. There is an actual standard for this which is quite long and which you are expected to follow when implementing a HTTP server or client.

python extract few lines from the output

I've created a python class to execute a curl command which in turn generates a token. The code works as expected.The script generates the expected output.Now, need few lines from the below mentioned output and not the entire output.
How do I extract the following lines from the output.
lines starting after
Set-Cookie: SESSION= till =;
. How do I extract these lines.
import requests
import json
from akamai.edgegrid import EdgeGridAuth, EdgeRc
from tabulate import tabulate
import os
import re
import base64
class generate_token(object):
"""
This library is to generate token for the test application
"""
def __init__(self,kwargs):
self.username = 'user'
self.password = 'abcdef'
self.contractName = kwargs.get("Names")
self.contractType = kwargs.get("Type")
self.accountId = kwargs.get("Id")
self.env = kwargs.get("env")
def requestSession(self):
if self.env == "test":
ping = os.popen('ping -c 1 hostname.com').read()
ip_addr = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",ping)[0]
else:
ping = os.popen('ping -c 1 different_hostname.com').read()
ip_addr = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",ping)[0]
url = "https://%s/EdgeAuth/asyncUserLogin" %(ip_addr)
command = "curl -is -v -k -d username=%s -d password=%s --request POST '%s' " %(self.username,self.password,url)
self.session = os.popen(command).read()
if __name__ == "__main__":
initData = {
'Names': 'names',
'Type': 'type',
'Id': 'ID',
'env' : 'test'
}
c = generate_token(kwargs=initData)
result = c.requestSession()
Entire output :
* TCP_NODELAY set
* WARNING: disabling hostname validation also disables SNI.
> POST /EdgeAuth/asyncUserLogin HTTP/1.1
> Host: 143.111.112.202
> User-Agent: curl/7.54.0
> Accept: */*
> Content-Length: 33
> Content-Type: application/x-www-form-urlencoded
>
} [33 bytes data]
* upload completely sent off: 33 out of 33 bytes
< HTTP/1.1 302 Moved Temporarily
< Date: Thu, 26 Jul 2018 15:54:07 GMT
< Server: Apache-Coyote/1.1
< Location: https://192.168.93.201/homeng/view/main
< Content-Length: 0
< Set-Cookie: SESSION=1-7FOmRFhEcci+8fAfh+DvLg==-1bk61eksykUHbWc9kx3LzL0KdkaxyDodxsEsu6siMR/1cpR7yLaD7HtTjXEr+XDowzpYcug4bfKIMjk2/T0aqJiHg5BrbRbnBeZnTcAUIpdyKGvtnmEQkSADoaxvgfYGrxqQctrVPdqJtPziqvIbjO1X8xh0Xtj/ZNvJXaSFs//w9ISNZ5hvIDBygUo+0EuYT8PSTYVPrcBCLaJUkC8ACg==-jdN937gosTsMx8fdaJqP5sA7wKPRgqv1RxRkX65SWeE=;
Path=/; Secure
< Via: 1.1 %{HTTP_HOST}
< X-Content-Type-Options: nosniff
< Content-Type: text/plain
Required output :
17FOmRFhEcci+8fAfh+DvLg==-1bk61eksykUHbWc9kx3LzL0KdkaxyDodxsEsu6siMR/1cpR7yLaD7HtTjXEr+XDowzpYcug4bfKIMjk2/T0aqJiHg5BrbRbnBeZnTcAUIpdyKGvtnmEQkSADoaxvgfYGrxqQctrVPdqJtPziqvIbjO1X8xh0Xtj/ZNvJXaSFs//w9ISNZ5hvIDBygUo+0EuYT8PSTYVPrcBCLaJUkC8ACg==-jdN937gosTsMx8fdaJqP5sA7wKPRgqv1RxRkX65SWeE=;

Don't use os.popen and friends to execute external programs - use the subprocess module for that:
ping = subprocess.check_output(['ping', '-c', '1', 'hostname.com'])
In the same way, don't execute a subprocess just to get a cookie from a url - you're already importing the excellent requests library in your code, why not use it!
r = requests.post(url, data={'username': self.username, 'password': self.password})
print (r.cookies['SESSION'])

Python MJPEG stream don't work outside VLC

I'm creating a streaming mjpeg from some IP cameras,the streaming with VLC works perfectly, just do not work inside a web page and other enterprise devices where other MJPEG streams taken from the internet work perfectly
here is my code:
class Streamer(Thread):
def __init__(self, *args, **kwargs):
self.container = kwargs.pop("container", None)
# self.q = kwargs.pop("queue", None)
# self.stream = kwargs.pop("stream", None)
super(Streamer, self).__init__(*args, **kwargs)
def run(self):
try:
print("dichiarazione handler")
server = ThreadedHTTPServer(('192.168.1.182', 8080), HandlerArguments(self.container))
print('started httpserver...')
server.serve_forever()
except KeyboardInterrupt:
print('^C received, shutting down server')
server.socket.close()
def HandlerArguments(init_args):
class MyHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.container = init_args
super(MyHandler, self).__init__(*args, **kwargs)
def do_GET(self):
print("GET")
if self.path.endswith('.mjpeg'):
self.container.stream = True
self.send_response(200)
self.send_header('Pragma:', 'no-cache')
self.send_header('Cache-Control:', 'no-cache')
self.send_header('Content-type:', 'multipart/x-mixed-replace;boundary=--myboundary')
self.end_headers()
try:
while self.container.stream == True:
# print(self.queue.qsize())
print("ciclo")
img = self.container.finalq.get()
ret, img_jpg = cv2.imencode(".jpeg", img, (cv2.IMWRITE_JPEG_QUALITY, 90))
self.wfile.write('--myboundary'.encode('utf-8'))
self.send_header('Content-type:','image/jpeg')
# self.send_header('Content-length:', str(len(img_jpg)))
self.end_headers()
img_str = img_jpg.tostring()
self.wfile.write(img_str)
# self.wfile.write('--myboundary\r\n'.encode('utf-8'))
# self.send_response(200)
# time.sleep(0.05)
except:
self.container.stream = False
while not self.container.finalq.empty():
self.container.finalq.get()
elif self.path.endswith('.jpg'):
self.send_response(200)
self.send_header('Pragma:', 'no-cache')
self.send_header('Cache-Control:', 'no-cache')
self.send_header('Content-type:','image/jpeg')
self.end_headers()
img = self.container.finalq.get()
ret, img_jpg = cv2.imencode(".jpeg", img, (cv2.IMWRITE_JPEG_QUALITY, 90))
# self.send_header('Content-length:', str(len(img_jpg)))
img_str = img_jpg.tostring()
self.wfile.write(img_str)
return MyHandler
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
#class ThreadedHTTPServer(HTTPServer):
"""Handle requests in a separate thread."""
I think the problem is in the request header, in particular this is a stream taken from the internet compared with my stream
curl -I http://85.90.40.19/mjpg/video.mjpg
HTTP/1.0 200 OK
Cache-Control: no-cache
Pragma: no-cache
Expires: Thu, 01 Dec 1994 16:00:00 GMT
Connection: close
Content-Type: multipart/x-mixed-replace; boundary=--myboundary
///////
curl -I http://192.168.1.182:8080/stream.mjpeg
HTTP/1.0 501 Unsupported method ('HEAD')
Server: BaseHTTP/0.6 Python/3.4.0
Date: Mon, 20 Jul 2015 10:00:54 GMT
Content-Type: text/html;charset=utf-8
Connection: close
Content-Length: 474
Especially in my request i see:
Content-Type: text/html;charset=utf-8
while in my code i run
self.send_header('Content-type:','image/jpeg')
Thanks in advance guys :)

curl -I sends a HTTP HEAD request.
In your code you're handling only the HTTP GET requests:
def do_GET(self):
See the error message: HTTP/1.0 501 Unsupported method ('HEAD')
Handle HEAD requests too:
def do_HEAD(self):
and output the expected headers since the clients may issue a HEAD before a GET.

asyncore python hangs

I try to do simple async http client with asyncore:
This code works fine and output is (fast enought):
www.gmail.com : recv http code: 301
www.yandex.ru : recv http code: 200
www.python.org : recv http code: 200
www.google.ru : recv http code: 200
www.gravatar.com : recv http code: 302
www.com.com : recv http code: 302
www.yahoo.com : recv http code: 302
www.bom.com : recv http code: 301
But than i uncomment line with not exist host:
#c = AsyncHTTP('http://www.no-such-host.ru') #!this line breaks execution!
The execution breaks, code hangs for some time, output part of data and hangs with no last data output:
connection error: [Errno -5] No address associated with hostname
www.gmail.com : recv http code: 301
www.yandex.ru : recv http code: 200
www.yahoo.com : recv http code: 302
www.com.com : recv http code: 302
www.bom.com : recv http code: 301
www.gravatar.com : recv http code: 302
...
some hosts are lost here and long delay at start.
Why this happen and how to fix this?
# coding=utf-8
import asyncore
import string, socket
import StringIO
import mimetools, urlparse
class AsyncHTTP(asyncore.dispatcher):
# HTTP requestor
def __init__(self, uri):
asyncore.dispatcher.__init__(self)
self.uri = uri
# turn the uri into a valid request
scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
assert scheme == "http", "only supports HTTP requests"
try:
host, port = string.split(host, ":", 1)
port = int(port)
except (TypeError, ValueError):
port = 80 # default port
if not path:
path = "/"
if params:
path = path + ";" + params
if query:
path = path + "?" + query
self.request = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host)
self.host = host
self.port = port
self.status = None
self.header = None
self.http_code = None
self.data = ""
# get things going!
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
#self.connect((host, port))
#return
try:
self.connect((host, port))
except Exception,e:
self.close()
self.handle_connect_expt(e)
def handle_connect(self):
self.send(self.request)
def handle_expt(self):
print "handle_expt error!"
self.close()
def handle_error(self):
print "handle_error error!"
self.close()
def handle_connect_expt(self,expt):
print "connection error:",expt
def handle_code(self):
print self.host," : ","recv http code: ",self.http_code
def handle_read(self):
data = self.recv(2048)
#print data
if not self.header:
self.data = self.data + data
try:
i = string.index(self.data, "\r\n\r\n")
except ValueError:
return # continue
else:
# parse header
fp = StringIO.StringIO(self.data[:i+4])
# status line is "HTTP/version status message"
status = fp.readline()
self.status = string.split(status, " ", 2)
self.http_code = self.status[1]
self.handle_code()
# followed by a rfc822-style message header
self.header = mimetools.Message(fp)
# followed by a newline, and the payload (if any)
data = self.data[i+4:]
self.data = ""
#header recived
#self.close()
def handle_close(self):
self.close()
c = AsyncHTTP('http://www.python.org')
c = AsyncHTTP('http://www.yandex.ru')
c = AsyncHTTP('http://www.google.ru')
c = AsyncHTTP('http://www.gmail.com')
c = AsyncHTTP('http://www.gravatar.com')
c = AsyncHTTP('http://www.yahoo.com')
c = AsyncHTTP('http://www.com.com')
c = AsyncHTTP('http://www.bom.com')
#c = AsyncHTTP('http://www.no-such-host.ru') #!this line breaks execution!
asyncore.loop()
ps: My system ubuntu 11.10 + python 2.7.2

You invoke a blocking name-resolution when you do self.connect((host, port)). Combined with your local DNS configuration, this is why your program has a long delay at startup.
An alternative to asyncore and figuring out how to do non-blocking name resolution yourself, you might think about using Twisted. Twisted's TCP connection setup API (mainly reactor.connectTCP or one of the APIs built on top of it) does not block. So a naive use of it will remain properly asynchronous.