#!/usr/bin/env python3 ## This Blog post is executable python code, it requires the gevent modules to run # pip as gevent, Debian as python3-gevent. import os import sys import gevent import gevent.socket as socket import gevent.ssl as ssl # we must use gevent's ssl module here, see ln 186 import signal from io import StringIO from email.utils import formatdate Fork = False # Welcome to Part 5.5 of this blog post series that might actually be a # book. Wherein we shall attempt to implement TLS correctly for this # our bespoke http implementation. Which if I did my research correctly. # Should look easy, but figureing out how not to shoot ourselves in # the foot took almost five hours of research. # A reminder please don't use this code in production. # If you want to fork it and make your own mistakes # after Part VI comes out be my guest, your own misfortune. # You should have also seen the prologue to the code. # which advises you to install mkcert, and tells you how to use it # to get a valid cert/keypair for use in this example. # If you didn't read it **READ IT NOW** CRLF = "\r\n" LF = "\n" class AccessDict(dict): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Convert nested dicts to AccessDict for key, value in self.items(): if isinstance(value, dict) and not isinstance(value, AccessDict): self[key] = AccessDict(value) def __getattr__(self, key): try: return self[key] except KeyError: raise AttributeError( f"'{type(self).__name__}' object has no attribute '{key}'" ) def __setattr__(self, key, value): # Convert nested dicts to AccessDict if isinstance(value, dict) and not isinstance(value, AccessDict): value = AccessDict(value) self[key] = value def __delattr__(self, key): try: del self[key] except KeyError: raise AttributeError( f"'{type(self).__name__}' object has no attribute '{key}'" ) # These are request and response objects similar to Go's standard # net/http package. Again this is still a toy implementation. # In a future part. I will go over how to make a more full featured # implementation. This is just making us ssl ready and laying the ground work class HttpRequest(AccessDict): def __init__(self, method="GET", path="/", headers={}, *args, **kwargs): super().__init__(*args, **kwargs) self["method"] = method self["headers"] = headers if "body" in kwargs: self["body"] = StringIO(kwargs["body"]) self["path"] = path if "Host" not in self["headers"]: self["headers"]["Host"] = "localhost" def read(self, seek): return self["body"].read(seek) def __str__(self): buf = StringIO() buf.write(f"{self.method} {self.path} HTTP/1.1") for k, v in self["headers"].items(): buf.write(f"{k}: {v}" + CRLF) buf.write(CRLF) buf.write(self["body"].getvalue() + CRLF) return buf.getvalue() + CRLF class HttpResponse(AccessDict): def __init__(self, status="400", headers={}, body="goodbye\r\n", *args, **kwargs): super().__init__(*args, **kwargs) self["status"] = status self["headers"] = headers self["body"] = StringIO() # We Must have date and host headers set correctly to use tls # so we unconditionally set them here if "host" not in kwargs: self["headers"]["Host"] = "localhost" else: self["headers"]["Host"] = kwargs["host"] self["headers"]["Date"] = formatdate(timeval=None, localtime=False, usegmt=True) self["headers"]["Content-Type"] = "text/plain; charset=UTF-8" def write(self, stuff): return self.body.write(stuff) # Foreshadowing (n): A literary device in which an author ... def __str__(self): buf = StringIO() print(self.headers) buf.write(f"HTTP/1.1 {self.status}" + CRLF) length = len(self["body"].getvalue()) for k, v in self["headers"].items(): buf.write(f"{k}: {v}\r\n") if "Content-Length" not in self["headers"]: buf.write(f"Content-Length: {length}\r\n") buf.write(CRLF) # Per RFC 9112 buf.write(self["body"].getvalue() + CRLF) return buf.getvalue() + CRLF RICKROLL_LYRICS = """ We're no strangers to love You know the rules and so do I A full commitment's what I'm thinkin' of You wouldn't get this from any other guy I just wanna tell you how I'm feeling Gotta make you understand Never gonna give you up, never gonna let you down Never gonna run around and desert you Never gonna make you cry, never gonna say goodbye Never gonna tell a lie and hurt you We've known each other for so long Your heart's been aching, but you're too shy to say it Inside, we both know what's been going on We know the game and we're gonna play it And if you ask me how I'm feeling Don't tell me you're too blind to see Never gonna give you up, never gonna let you down Never gonna run around and desert you Never gonna make you cry, never gonna say goodbye Never gonna tell a lie and hurt you Never gonna give you up, never gonna let you down Never gonna run around and desert you Never gonna make you cry, never gonna say goodbye Never gonna tell a lie and hurt you """ head_response = HttpResponse() head_response.status = 200 head_response.headers["Content-Length"] = 980 head_response.write("") good_response = HttpResponse() good_response.status = 200 good_response.headers["Last-Modified"] = "Mon, 27 July 1987 00:00 GMT" good_response.write(RICKROLL_LYRICS) error_response = HttpResponse() error_response.status = 405 # a 405 here is closer to RFC compliant error_response.write("Together forever and never to part Together forever we two") client_procs = [] # commentary on ln CHANGEME ct_svr_proc = None ssl_svr_proc = None # You've Seen all this before, in the last Part 1. I will shorten commentary class NullDevice: def write(self, s): pass def hup_handle(sig, fr): sys.exit() # We've been using gevent all along, but now it's time to say the quiet part # outloud. Gevent is an alternative concurrency module for python. # Trying to use gevent and standard python's stuff side by side # is a quick road to madness. I'll explain more in the Prequal series # For now it's enough to know the APIs are identical. # With that out of the way, we come to our first real decision, # that has security implications. # We could implement TLS in two ways. Method 1 we have the server. # listen on two different ports. As http does on 80/443 # the other approach involves encrypting traffic on the port we # already use. The one port method may seem safer. # This was the route chosen by the gemini project. # But for teaching purposes the two port method works better. # so that's what we'll do. # This requires a couple of changes, in server handler. # first we change it's name, and make the corresponding change # in main, and we'll copy it almost verbatim, and make changes for TLS def cleartext_server_handler(): serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) serversock.bind(("", 1337)) serversock.listen(10) while True: client, addr = serversock.accept() print(addr) client_procs.append(gevent.spawn(client_handler, client)) gevent.sleep(0.25) serversock.close() return # TLS looks easy, but in practice almost every library for handling it # in any language you can name is... HOT GARBAGE. Python is not the # exception, but i find it does have less security foot guns. # if anything screws up it will crash out with an SSLError # it's error messages are cryptic. # But once you've done it properly it looks easy. # This took about four hours to debug, but i've got it finally def tls_server_handler(): # the context is sort of like a container for cryptographic settings # we load the default context, which contains the best default # settings as reviewed, by the python security people.abs # this avoids a lot of foot guns # Note here that the server/client is reversed # Because we are a server we need the context # for clients. ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) # Remember what i said about best default and not shooting # ourselves in the foot. Well Firefox, does a stupid thing # during TLS handshake and declares it supports. SSLv3 # Which has been considered hopelessly broken since 2014. # Mozilla's own security people even said so at the time. # See References. Anyway Setting the minimum and maximum # explicitly to TLSv1.2/1.3 avoids this wrongness. # so we do it ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ctx.maximum_version = ssl.TLSVersion.TLSv1_3 serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) serversock.bind(("", 1972)) # we choose another port number, 1972 foreshadows # a bit # Next we load our key and cert # I'm assuming you used the mkcert method # if you used the self signed method just place your cert/key # See comment on ln 277 ctx.load_cert_chain(certfile="cert.pem", keyfile="key.pem") serversock.listen(10) while True: client, addr = serversock.accept() print(addr) try: secure = ctx.wrap_socket(client, server_side=True) # last step in the process is to wrap the client socket in # TLS. The SSLContext does this for us. # but you must pass server_side=True to avoid silly defaults client_procs.append(gevent.spawn(client_handler, secure)) gevent.sleep(0.25) # If anything goes wrong here the ssl.SSLError is thrown # i was originally going to leave it to crash, but # the browser behavior on pki errors also causes this # exception. So we will log the error and continue. except ssl.SSLError as e: print(e) gevent.sleep(0.25) continue finally: gevent.sleep(0.25) serversock.close() return # One Change here see ln 325 def client_handler(sock): print("Client handler spawn") junk_counter = 0 while True: if junk_counter > 3: sock.close() return data = sock.recv(4096) dstring = data.decode("UTF-8") if dstring.startswith("GET"): break elif dstring.startswith("HEAD"): hr = str(head_response) sock.send(hr.encode("utf-8")) sock.shutdown(socket.SHUT_RDWR) sock.close() return else: error = str(error_response) sock.send(error.encode("utf-8")) junk_counter += 1 gevent.sleep(0.25) # this is a somewhat magical value, see Part II default = str(good_response) sock.send(default.encode("utf-8")) sock.shutdown(socket.SHUT_RDWR) # we do a more graceful exit here by # shutting down the socket, makes things faster for TLS # may have an effect on client response time to but i didn't notice it. sock.close() return def daemon_main(): svr_proc = gevent.spawn(cleartext_server_handler) ssl_svr_proc = gevent.spawn(tls_server_handler) client_procs.append(svr_proc) client_procs.append(ssl_svr_proc) gevent.joinall(client_procs) sys.exit(0) # so things will not fork while i'm debbuging if not Fork: daemon_main() pid = os.fork() if pid: os._exit(0) else: os.setpgrp() os.umask(0) print(os.getpid()) sys.stdout = NullDevice() sys.stderr = NullDevice() signal.signal(signal.SIGHUP, hup_handle) signal.signal(signal.SIGTERM, hup_handle) daemon_main() # To recap we just did a bunch of work, for no user visible change # This is not a bad thing, often the first drafts of programs. # Will fit the requirements of the moment. But when the requirements # change the program must be adapted to fit. # This process of iteration and redesign, # is called "paying down technical debt", and it should be done whenever # possible. # # And we've just moved up to the second level of the 7 story mountain # Yay us. # # References # Robustness Principal (Devopedia): https://devopedia.org/postel-s-law # IETF RFC 9112 HTTP/1.1 https://datatracker.ietf.org/doc/html/rfc9112 #