Develop Reference

Get a SSL client certificate from server side in python?

I know that during ssl handshaking the server can optionally request the client certificate.
In Python, my client can get the server certificate by ssl.get_server_certificate method but I don't find a similar method that permits to get the client certificate from the server side.
I checked this blog but requires to have the client.crt already at the server side. My intention is getting this self-signed client.crt directly from the client peer. It's important that all the certificates must be self-signed so I don't want CA verifications.
Should I use getpeercert() ? How exactly ?
Here is my part of code if you need it:
client
certfile = 'client.crt'
keyfile = 'client.key'
server_certificate = ssl.get_server_certificate((HOST,PORT),timeout=10)
with open('server.crt', "wt") as f:
f.write(certificate)
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile='server.crt')
context.load_cert_chain(certfile=certfile, keyfile=keyfile)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client = context.wrap_socket(client, server_hostname=HOST)
server
certfile = 'server.crt'
keyfile = 'server.key'
contextInstance = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
contextInstance.check_hostname = False
contextInstance.verify_mode = ssl.CERT_REQUIRED
contextInstance.load_cert_chain(
certfile="server.crt", keyfile="server.key" )
# how can I get the client.crt to put here ?
#contextInstance.load_verify_locations(cafile="client.crt")
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind((HOST, PORT))
server_socket.listen()
while True:
try:
client_socket, client_address = server_socket.accept()
conn = contextInstance.wrap_socket(client_socket, server_side=True)
print("SSL established. Peer: {}".format(conn.getpeercert()))

Python3 NAT hole punching

I know this topic is not new. There is various information out there although, the robust solution is not presented (at least I did not found). I have a P2P daemon written in python3 and the last element on the pie is to connect two clients behind the NAT via TCP. My references for this topic:
https://bford.info/pub/net/p2pnat/
How to make 2 clients connect each other directly, after having both connected a meeting-point server?
Problems with TCP hole punching
What I have done so far:
SERVER:
#!/usr/bin/env python3
import threading
import socket
MY_AS_SERVER_PORT = 9001
TIMEOUT = 120.0
BUFFER_SIZE = 4096
def get_my_local_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return bytes(IP, encoding='utf-8')
def wait_for_msg(new_connection, client_address):
while True:
try:
packet = new_connection.recv(BUFFER_SIZE)
if packet:
msg_from_client = packet.decode('utf-8')
client_connected_from_ip = client_address[0]
client_connected_from_port = client_address[1]
print("We have a client. Client advertised his local IP as:", msg_from_client)
print(f"Although, our connection is from: [{client_connected_from_ip}]:{client_connected_from_port}")
msg_back = bytes("SERVER registered your data. Your local IP is: " + str(msg_from_client) + " You are connecting to the server FROM: " + str(client_connected_from_ip) + ":" + str(client_connected_from_port), encoding='utf-8')
new_connection.sendall(msg_back)
break
except ConnectionResetError:
break
except OSError:
break
def server():
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock.bind((get_my_local_ip().decode('utf-8'), MY_AS_SERVER_PORT))
sock.listen(8)
sock.settimeout(TIMEOUT)
while True:
try:
new_connection, client_address = sock.accept()
if new_connection:
threading.Thread(target=wait_for_msg, args=(new_connection,client_address,)).start()
# print("connected!")
# print("")
# print(new_connection)
# print("")
# print(client_address)
msg = bytes("Greetings! This message came from SERVER as message back!", encoding='utf-8')
new_connection.sendall(msg)
except socket.timeout:
pass
if __name__ == '__main__':
server()
CLIENT:
#!/usr/bin/python3
import sys
import socket
import time
import threading
SERVER_IP = '1.2.3.4'
SERVER_PORT = 9001
# We don't want to establish a connection with a static port. Let the OS pick a random empty one.
#MY_AS_CLIENT_PORT = 8510
TIMEOUT = 3
BUFFER_SIZE = 4096
def get_my_local_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return bytes(IP, encoding='utf-8')
def constantly_try_to_connect(sock):
while True:
try:
sock.connect((SERVER_IP, SERVER_PORT))
except ConnectionRefusedError:
print(f"Can't connect to the SERVER IP [{SERVER_IP}]:{SERVER_PORT} - does the server alive? Sleeping for a while...")
time.sleep(1)
except OSError:
#print("Already connected to the server. Kill current session to reconnect...")
pass
def client():
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
#sock.bind((get_my_local_ip().decode('utf-8'), MY_AS_CLIENT_PORT))
sock.settimeout(TIMEOUT)
threading.Thread(target=constantly_try_to_connect, args=(sock,)).start()
while True:
try:
packet = sock.recv(BUFFER_SIZE)
if packet:
print(packet)
sock.sendall(get_my_local_ip())
except OSError:
pass
if __name__ == '__main__':
client()
Now the current code results:
./tcphole_server.py
We have a client. Client advertised his local IP as: 10.10.10.50
Although, our connection is from: [89.22.11.50]:32928
We have a client. Client advertised his local IP as: 192.168.1.20
Although, our connection is from: [78.88.77.66]:51928
./tcphole_client1.py
b'Greetings! This message came from SERVER as message back!'
b'SERVER registered your data. Your local IP is: 192.168.1.20 You are connecting to the server FROM: 89.22.11.50:32928'
./tcphole_client2.py
b'Greetings! This message came from SERVER as message back!'
b'SERVER registered your data. Your local IP is: 10.10.10.50 You are connecting to the server FROM: 78.88.77.66:51928'
As you can see the server has all information to connect two clients. We can send details about the other peer individually through the current server-client connection.
Now two questions remain in my head:
Assuming the SERVER sends information about CLIENT 1 and CLIENT 2 for each of the peers. And now the CLIENTS starts connecting like [89.22.11.50]:32928 <> [78.88.77.66]:51928 Does the SERVER should close the current connections with the CLIENTS?
How the CLIENT Router behaves? I assume it expecting the same EXTERNAL SERVER SRC IP [1.2.3.4], instead gets one of the CLIENTS EXT IP for instance [89.22.11.50] or [78.88.77.66]?
This is messier than I thought. Any help to move forward appreciated. Hope this would help other Devs/DevOps too.

Finally found the expected behavior! Don't want to give too much code here but I hope after this you will understand the basics of how to implement it. Best to have a separate file in each of the client's folder - nearby ./tcphole_client1.py and ./tcphole_client2.py. We need to connect fast after we initiated sessions with the SERVER. Now for instance:
./tcphole_client_connector1.py 32928 51928
./tcphole_client_connector2.py 51928 32928
Remember? We need to connect to the same ports as we initiated with SERVER:
[89.22.11.50]:32928 <> [78.88.77.66]:51928
The first port is needed to bind the socket (OUR). With the second port, we are trying to connect to the CLIENT. The other CLIENT doing the same procedure except it binds to his port and connects to yours bound port. If the ROUTER still has an active connection - SUCCESS.

What details about the host machine does python send to the web server while establishing a connection through sockets?

I am building a passive reconnaissance tool and one of it functionality is to get certificate info about a domain.
What details of my machine are being to sent to the webserver?
The below code is used to get the certificate info.
from OpenSSL import SSL
from cryptography import x509
from cryptography.x509.oid import NameOID
import idna
from socket import socket
from collections import namedtuple
HostInfo = namedtuple(field_names='cert hostname peername', typename='HostInfo')
HOSTS = [
('google.com', 443),
('yahoo.com', 443),
('yahoo.com', 443),
]
def get_certificate(hostname, port):
hostname_idna = idna.encode(hostname)
sock = socket()
sock.connect((hostname, port))
peername = sock.getpeername()
ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible
ctx.check_hostname = False
ctx.verify_mode = SSL.VERIFY_NONE
sock_ssl = SSL.Connection(ctx, sock)
sock_ssl.set_connect_state()
sock_ssl.set_tlsext_host_name(hostname_idna)
sock_ssl.do_handshake()
cert = sock_ssl.get_peer_certificate()
crypto_cert = cert.to_cryptography()
sock_ssl.close()
sock.close()
return HostInfo(cert=crypto_cert, peername=peername, hostname=hostname)
import concurrent.futures
if __name__ == '__main__':
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as e:
for hostinfo in e.map(lambda x: get_certificate(x[0], x[1]), HOSTS):
print_basic_info(hostinfo)
Is my IP address being sent?
Is python sending any user-agent and what other details are being sent while establishing a connection?

While Python doesn't explicitly send your IP, it is always sent with any TCP request. If it isn't sent, you cannot receive the response from the server.
According to http://evanhahn.com/python-requests-library-useragent, the useragent is
python-requests/{package version} {runtime}/{runtime version} {uname}/{uname -r} when using python-requests, but you can override it by setting the request header.
However, you are not using python-requests but raw sockets, so no data is transferred except what you tell it to (and your IP and some metadata for the TCP connection)

Is there an easy way to check if a website has an SSL certificate

Using urllib2/SSL?
I have a large list of domains, and it would be useful to know if any had ssl certs that I wasn't aware of.

I'm not sure about using urllib2. Unfortunately, the closest information I can find on this is this link and the associated code:
import socket
import ssl
HOST = "www.google.com"
PORT = 443
# replace HOST name with IP, this should fail connection attempt
HOST = socket.getaddrinfo(HOST, PORT)[0][4][0]
print(HOST)
# create socket and connect to server
# server address is specified later in connect() method
sock = socket.socket()
sock.connect((HOST, PORT))
# wrap socket to add SSL support
sock = ssl.wrap_socket(sock,
# flag that certificate from the other side of connection is required
# and should be validated when wrapping
cert_reqs=ssl.CERT_REQUIRED,
# file with root certificates
ca_certs="cacerts.txt"
)
# security hole here - there should be an error about mismatched host name
# manual check of hostname
cert = sock.getpeercert()
for field in cert['subject']:
if field[0][0] == 'commonName':
certhost = field[0][1]
if certhost != HOST:
raise ssl.SSLError("Host name '%s' doesn't match certificate host '%s'"
% (HOST, certhost))
Although the comments in the file are fairly extensive, the wiki in the first link also lists these instructions:
To validate that a certificate matches requested site, you need to check commonName field in the subject of the certificate. This information can be accessed with getpeercert() method of wrapped socket.
You will need cacerts.txt file that contains root certificates placed alongside the script - see below how to get an updated list. To check that certificate validation works - use https://www.debian-administration.org/ in HOST name. This site's certificate is not signed by any root certificates from cacerts.txt, so you will get an error.
You could look into the pyopenssl module as well, since according to the first link I posted, it can be used to validate SSL certificates in this way:
import socket
from OpenSSL import SSL
HOST = "www.google.com"
PORT = 443
# replace HOST name with IP, this should fail connection attempt,
# but it doesn't by default
HOST = socket.getaddrinfo(HOST, PORT)[0][4][0]
print(HOST)
# uses HOST
def verify_cb(conn, x509, errno, errdepth, retcode):
"""
callback for certificate validation
should return true if verification passes and false otherwise
"""
if errno == 0:
if errdepth != 0:
# don't validate names of root certificates
return True
else:
if x509.get_subject().commonName != HOST:
return False
else:
return False
context = SSL.Context(SSL.SSLv23_METHOD)
context.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verify_cb)
context.load_verify_locations("cacerts.txt")
# create socket and connect to server
sock = socket.socket()
sock = SSL.Connection(context, sock)
sock.connect((HOST, PORT))
sock.do_handshake()
According to the documentation in the first link, you will need the latest certificate versions from here for these examples.

HTTPS proxy tunneling with the ssl module

I'd like to manually (using the socket and ssl modules) make an HTTPS request through a proxy which itself uses HTTPS.
I can perform the initial CONNECT exchange just fine:
import ssl, socket
PROXY_ADDR = ("proxy-addr", 443)
CONNECT = "CONNECT example.com:443 HTTP/1.1\r\n\r\n"
sock = socket.create_connection(PROXY_ADDR)
sock = ssl.wrap_socket(sock)
sock.sendall(CONNECT)
s = ""
while s[-4:] != "\r\n\r\n":
s += sock.recv(1)
print repr(s)
The above code prints HTTP/1.1 200 Connection established plus some headers, which is what I expect. So now I should be ready to make the request, e.g.
sock.sendall("GET / HTTP/1.1\r\n\r\n")
but the above code returns
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>400 Bad Request</title>
</head><body>
<h1>Bad Request</h1>
<p>Your browser sent a request that this server could not understand.<br />
Reason: You're speaking plain HTTP to an SSL-enabled server port.<br />
Instead use the HTTPS scheme to access this URL, please.<br />
</body></html>
This makes sense too, since I still need to do an SSL handshake with the example.com server to which I'm tunneling. However, if instead of immediately sending the GET request I say
sock = ssl.wrap_socket(sock)
to do the handshake with the remote server, then I get an exception:
Traceback (most recent call last):
File "so_test.py", line 18, in <module>
ssl.wrap_socket(sock)
File "/usr/lib/python2.6/ssl.py", line 350, in wrap_socket
suppress_ragged_eofs=suppress_ragged_eofs)
File "/usr/lib/python2.6/ssl.py", line 118, in __init__
self.do_handshake()
File "/usr/lib/python2.6/ssl.py", line 293, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [Errno 1] _ssl.c:480: error:140770FC:SSL routines:SSL23_GET_SERVER_HELLO:unknown protocol
So how can I do the SSL handshake with the remote example.com server?
EDIT: I'm pretty sure that no additional data is available before my second call to wrap_socket because calling sock.recv(1) blocks indefinitely.

This should work if the CONNECT string is rewritten as follows:
CONNECT = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n\r\n" % (server, port)
Not sure why this works, but maybe it has something to do with the proxy I'm using. Here's an example code:
from OpenSSL import SSL
import socket
def verify_cb(conn, cert, errun, depth, ok):
return True
server = 'mail.google.com'
port = 443
PROXY_ADDR = ("proxy.example.com", 3128)
CONNECT = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n\r\n" % (server, port)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(PROXY_ADDR)
s.send(CONNECT)
print s.recv(4096)
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.set_verify(SSL.VERIFY_PEER, verify_cb)
ss = SSL.Connection(ctx, s)
ss.set_connect_state()
ss.do_handshake()
cert = ss.get_peer_certificate()
print cert.get_subject()
ss.shutdown()
ss.close()
Note how the socket is first opened and then open socket placed in SSL context. Then I manually initialize SSL handshake. And output:
HTTP/1.1 200 Connection established
<X509Name object '/C=US/ST=California/L=Mountain View/O=Google Inc/CN=mail.google.com'>
It's based on pyOpenSSL because I needed to fetch invalid certificates too and Python built-in ssl module will always try to verify the certificate if it's received.

Judging from the API of the OpenSSL and GnuTLS library, stacking a SSLSocket onto a SSLSocket is actually not straightforwardly possible as they provide special read/write functions to implement the encryption, which they are not able to use themselves when wrapping a pre-existing SSLSocket.
The error is therefore caused by the inner SSLSocket directly reading from the system socket and not from the outer SSLSocket. This ends in sending data not belonging to the outer SSL session, which ends badly and for sure never returns a valid ServerHello.
Concluding from that, I would say there is no simple way to implement what you (and actually myself) would like to accomplish.

Finally I got somewhere expanding on #kravietz and #02strich answers.
Here's the code
import threading
import select
import socket
import ssl
server = 'mail.google.com'
port = 443
PROXY = ("localhost", 4433)
CONNECT = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n\r\n" % (server, port)
class ForwardedSocket(threading.Thread):
def __init__(self, s, **kwargs):
threading.Thread.__init__(self)
self.dest = s
self.oursraw, self.theirsraw = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
self.theirs = socket.socket(_sock=self.theirsraw)
self.start()
self.ours = ssl.wrap_socket(socket.socket(_sock=self.oursraw), **kwargs)
def run(self):
rl, wl, xl = select.select([self.dest, self.theirs], [], [], 1)
print rl, wl, xl
# FIXME write may block
if self.theirs in rl:
self.dest.send(self.theirs.recv(4096))
if self.dest in rl:
self.theirs.send(self.dest.recv(4096))
def recv(self, *args):
return self.ours.recv(*args)
def send(self, *args):
return self.outs.recv(*args)
def test():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(PROXY)
s = ssl.wrap_socket(s, ciphers="ALL:aNULL:eNULL")
s.send(CONNECT)
resp = s.read(4096)
print (resp, )
fs = ForwardedSocket(s, ciphers="ALL:aNULL:eNULL")
fs.send("foobar")
Don't mind custom cihpers=, that only because I didn't want to deal with certificates.
And there's depth-1 ssl output, showing CONNECT, my response to it ssagd and depth-2 ssl negotiation and binary rubbish:
[dima#bmg ~]$ openssl s_server -nocert -cipher "ALL:aNULL:eNULL"
Using default temp DH parameters
Using default temp ECDH parameters
ACCEPT
-----BEGIN SSL SESSION PARAMETERS-----
MHUCAQECAgMDBALAGQQgmn6XfJt8ru+edj6BXljltJf43Sz6AmacYM/dSmrhgl4E
MOztEauhPoixCwS84DL29MD/OxuxuvG5tnkN59ikoqtfrnCKsk8Y9JtUU9zuaDFV
ZaEGAgRSnJ81ogQCAgEspAYEBAEAAAA=
-----END SSL SESSION PARAMETERS-----
Shared ciphers: [snipped]
CIPHER is AECDH-AES256-SHA
Secure Renegotiation IS supported
CONNECT mail.google.com:443 HTTP/1.0
Connection: close
sagq
�u\�0�,�(�$��
�"�!��kj98���� �m:��2�.�*�&���=5�����
��/�+�'�#�� ����g#32��ED���l4�F�1�-�)�%���</�A������
�� ������
�;��A��q�J&O��y�l

It doesn't sound like there's anything wrong with what you're doing; it's certainly possible to call wrap_socket() on an existing SSLSocket.
The 'unknown protocol' error can occur (amongst other reasons) if there's extra data waiting to be read on the socket at the point you call wrap_socket(), for instance an extra \r\n or an HTTP error (due to a missing cert on the server end, for instance). Are you certain you've read everything available at that point?
If you can force the first SSL channel to use a "plain" RSA cipher (i.e. non-Diffie-Hellman) then you may be able to use Wireshark to decrypt the stream to see what's going on.

Building on #kravietz answer. Here is a version that works in Python3 through a Squid proxy:
from OpenSSL import SSL
import socket
def verify_cb(conn, cert, errun, depth, ok):
return True
server = 'mail.google.com'
port = 443
PROXY_ADDR = ("<proxy_server>", 3128)
CONNECT = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n\r\n" % (server, port)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(PROXY_ADDR)
s.send(str.encode(CONNECT))
s.recv(4096)
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.set_verify(SSL.VERIFY_PEER, verify_cb)
ss = SSL.Connection(ctx, s)
ss.set_connect_state()
ss.do_handshake()
cert = ss.get_peer_certificate()
print(cert.get_subject())
ss.shutdown()
ss.close()
This works in Python 2 also.