don't commit those

.gitignore

@@ -0,0 +1 @@
+*.pem

localhost.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDDzCCAfegAwIBAgIUCbq4bh+c1XB+XJT+Bkoazr0rLfAwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI1MDkwODA0MzUxM1oXDTI1MTAw
+ODA0MzUxM1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAm3v5FpenJ45pM6gA7KfS7+AkIxHAyxKgbnfC8T5yIueR
+J8XcLtEJbV3pxD+q6WtXfZGVQyGKGmhLxp5dIB3FhEJsEekjnebLLHEsRuo5M9r0
+9eezjCzzeat0z9UYV8agAcM2ONdo6GzIBzU4G6TaHlVM6Z/VYkxKTHnAC3HyS5oT
+gpExZnFfD1KL3usVsk33YJttEqLvWAsoOT881UpGWrd3c2b2l2uWXpNRO7hUHJ+/
+Dw36fGK+OJj1/Ivi0/wCNZZ1e7JUWs13kbrBJDkfUBJ7Gvug0H6ufzsLYgXdGuTu
+siCKEFSYHuhvBbzx8z1wIOKEvD7pxxLrKYJ/RgRpNwIDAQABo1kwVzAUBgNVHREE
+DTALgglsb2NhbGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMB
+MB0GA1UdDgQWBBS7KJzA5Ou2ow47Fqphh6lxjj7ieDANBgkqhkiG9w0BAQsFAAOC
+AQEAIaM7L8K0z/V2c3oUP2MF4+Z4G7TARx2AQEUOKEgb+jxCPj2p1/vxChrekvbx
+RQnakvmOmNyW+6Omj+DN7DwcXCA4604Et3aec2Br/1XcaEwOHkea9BKMufOgtfED
+PDdMhuPdki1GDqIUCeyAS3MikqczUjvZ+ZsDaBVOrfOh3oruX5F7CDt1YWi6GxPB
+NXVbnvEJCJzqj+jqXBUBxerALOiIwHFstqrEXubO4zPrAiU+3kCLDXwTGmLfkXqE
+wkFyHhs/mru9OywkDL4Y5eL9oV1b6EYpCR8E9Yonqw3o0+WsIQ758MWuiPrrhwqG
+qtKdMu75VQvcfw+GeBBYh5Yl+w==
+-----END CERTIFICATE-----

localhost.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCbe/kWl6cnjmkz
+qADsp9Lv4CQjEcDLEqBud8LxPnIi55Enxdwu0QltXenEP6rpa1d9kZVDIYoaaEvG
+nl0gHcWEQmwR6SOd5ssscSxG6jkz2vT157OMLPN5q3TP1RhXxqABwzY412jobMgH
+NTgbpNoeVUzpn9ViTEpMecALcfJLmhOCkTFmcV8PUove6xWyTfdgm20Sou9YCyg5
+PzzVSkZat3dzZvaXa5Zek1E7uFQcn78PDfp8Yr44mPX8i+LT/AI1lnV7slRazXeR
+usEkOR9QEnsa+6DQfq5/OwtiBd0a5O6yIIoQVJge6G8FvPHzPXAg4oS8PunHEusp
+gn9GBGk3AgMBAAECggEAJUlvPDxFIUbVMsac5iP/AXiSofhdcDW41JUS4nCzxWwc
+EtovyehmZtxpNZ+BGLYdxqa1kWJHMLsHRQvwUEXjeqrFKOxslq7k1XUUhbMg4a8m
+JJyaocib4Pc9raOwUUh4UcPjSnFaHrcLpzLbkEGR70lMhxBGB2s9PCbMZ9I9JWTf
+SmgO9GHQ8GR6QPqlF9ZameyHCX4D68bQBdihOzd6x2ek4z0ebK2/IupLN8CKBeNw
+mZYh9XqDETwe67iB+3uVAbogz7KBiJ2oeRcSb8PA0XqkOsNLWgXDQTJ6uUtMuTil
+UTMEIivYuszpEk7VOnR0Nj5Rnsu+K1QuyMwpQCX8xQKBgQDWyUMTEUh8Y7ZxeqFX
+Zl1Tfck2113cUblbweO+OmMi8pqxqFNqlBnqxXnj033gSfm2zrI38F33B9Jyhvzq
+e3gnq31zq3LwKQbaczhAX0g93Bsm9h3QL6N+XY8COkiYkePyAAt/eENQZpfBJi3O
+NpJhLRR6bbmFRwmkVzrrd1zxdQKBgQC5Ua3wQuM65wHv51LOiWPAuHG3/pPdHvPG
+o/YWqWu2Hr/uUCd7RYV0bxvBJWqY2y80/Nadt3HQNdzRd8ygmWInF0TW0HqIFZwV
+LbK7sBi4pW9wAdqMFr5BQnk1W6rDeNTWD8K+8lbd4d+OjgtZC4CK20xR6NugBH9m
+W0J0LYgOewKBgQCiAhM22a18LdYaiG4UN6EjbdiNJiulGHugy4HWJcJLRQUMBjRN
+SsK1xBhpkUf8GrBhhE0HRqYJw/un6UvyLgl2mrK4wdSjc764nXoLjBM4ncJZRAE+
+3AANO9K30nCZrElsaz5A+tyDU68ZwIuCZMVKyS8OHZ92+Rs7u5Q0sccIVQKBgD4A
+C7eESU2dl9JRjCy5XnxNuQ4byBCEmH5uwJhYWkb2BrSOcIcXfUy1F44JHJ7DRgnu
+RUdC5nsIajZSZE2ew23cpRVRbo003aFgRpnwknTENII+vIV93m0q9i5Z2snHFT4A
+y+DiZxmYxhiFgVprNLhAIkqNI11n48+03IjN6uUdAoGAZaxjuUgej7wdSfGr15IH
+w/YEtuKid3BnBo9q3PbZ36Nop/Ih81XZb4+uis7PNMC/GuAZE6h+6RkJhALVWZOy
+BPmsHqwgy+H4N/BdgSPpKTtgkq4NdtHqrVAAmh9JFfFJBFuf8iYM60dafTJh6ijI
+iiR8hF7pL4RRHe+kxX7v8Vg=
+-----END PRIVATE KEY-----

rohttp.py

@@ -33,10 +33,10 @@ Fork = False
 # i suspect this one might be ai generated.
 # It's a subclass of dict with key access via the dot
 # operator. Similar to how ruby does things
-# I don't know why this isn't the default
-# i've coded this dozens of times but i couldn't use
-# client's code in a public project.
-# Thus cut 'n' paste.
+
+# Obviously some things still need standard dictionary access
+# i found this code looked at it, tested it, before using it
+#
 
 CRLF = "\r\n"
 LF = "\n"
@@ -78,13 +78,12 @@ class AccessDict(dict):
 # In a future part. I will go over how to make a more full featured
 # implementation. This is just making us ssl ready and laying the ground work
 class HttpRequest(AccessDict):
-    def __init__(
-        self, method="GET", path="/", headers={}, body="goodbye\r\n", *args, **kwargs
-    ):
+    def __init__(self, method="GET", path="/", headers={}, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self["method"] = method
         self["headers"] = headers
-        self["body"] = StringIO(body)
+        if "body" in kwargs:
+            self["body"] = StringIO(kwargs["body"])
         self["path"] = path
         if "Host" not in self["headers"]:
             self["headers"]["Host"] = "localhost"
@@ -96,14 +95,14 @@ class HttpRequest(AccessDict):
         buf = StringIO()
         buf.write(f"{self.method} {self.path} HTTP/1.1")
         for k, v in self["headers"].items():
-            buf.write(f"{k}: {v}\r\n")
-        buf.write(CRLF + CRLF)
-        buf.write(self["body"].getvalue() + "\r\n")
-        return buf.getvalue() + "\r\n"
+            buf.write(f"{k}: {v}" + CRLF)
+        buf.write(CRLF)
+        buf.write(self["body"].getvalue() + CRLF)
+        return buf.getvalue() + CRLF
 
 
 class HttpResponse(AccessDict):
-    def __init__(self, status="404", headers={}, body="goodbye\r\n", *args, **kwargs):
+    def __init__(self, status="400", headers={}, body="goodbye\r\n", *args, **kwargs):
         super().__init__(*args, **kwargs)
         self["status"] = status
         self["headers"] = headers
@@ -125,15 +124,16 @@ class HttpResponse(AccessDict):
     def __str__(self):
         buf = StringIO()
         print(self.headers)
-        buf.write(f"HTTP/1.1 {self.status}\r\n")
+        buf.write(f"HTTP/1.1 {self.status}" + CRLF)
         length = len(self["body"].getvalue())
         for k, v in self["headers"].items():
             buf.write(f"{k}: {v}\r\n")
+        if "Content-Length" not in self["headers"]:
             buf.write(f"Content-Length: {length}\r\n")
-        buf.write(CRLF + CRLF)  # Per RFC 9112
+        buf.write(CRLF)  # Per RFC 9112
 
-        buf.write(self["body"].getvalue() + "\r\n")
-        return buf.getvalue() + "\r\n"
+        buf.write(self["body"].getvalue() + CRLF)
+        return buf.getvalue() + CRLF
 
 
 RICKROLL_LYRICS = """
@@ -170,6 +170,10 @@ Never gonna make you cry, never gonna say goodbye
 Never gonna tell a lie and hurt you
 
 """
+head_response = HttpResponse()
+head_response.status = 200
+head_response.headers["Content-Length"] = 980
+head_response.write("")
 good_response = HttpResponse()
 good_response.status = 200
 good_response.headers["Last-Modified"] = "Mon, 27 July 1987 00:00 GMT"
@@ -209,7 +213,7 @@ def server_handler():
     return
 
 
-# I made two simple changes to make it use our new http objects
+# I made three simple changes to make it use our new http objects
 # This will still accept anything as long as the method verb
 # is correct. Clients need not conform to RFC 9112 (yet).
 # We however, do our best effort to conform to rfc 9112,
@@ -219,8 +223,19 @@ def server_handler():
 # the client, but strict in what you send back, was first
 # was first forumlated by John Postel in that later half
 # of the 1970s.
+#
+# Lastly we add support for the HEAD method, as some http clients
+# Will get confused if we don't have it
+# RFC 9112 appears to say GET and HEAD are the only methods we
+# are ABSOLUTELY REQUIRED to support. So we add it.abs
+# At this point it will get a static response as well.
+# See Above.
 
-# Doing this, is also motivation for me to write Parts 4, 5, and 6
+# Doing this, is also motivation for me to write Parts 1, 2, and 3
+# By the way you're in a Star Wars, Sort of thing
+# This is Part 5. I thought the 4 would be more entertaining.abs
+# Parts 7, 8, and 9 will be made for Capitalism reasons
+# AKA Donate on my kofi link at the end.
 
 
 def client_handler(sock):
@@ -234,11 +249,16 @@ def client_handler(sock):
         dstring = data.decode("UTF-8")
         if dstring.startswith("GET"):
             break
+        elif dstring.startswith("HEAD"):
+            hr = str(head_response)
+            sock.send(hr.encode("utf-8"))
+            sock.close()
+            return
         else:
             error = str(error_response)
             sock.send(error.encode("utf-8"))
             junk_counter += 1
-        gevent.sleep(0.25)
+        gevent.sleep(0.25)  # this is a somewhat magical value
     default = str(good_response)
     sock.send(default.encode("utf-8"))
     sock.close()

rohttptls.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python3
+## This Blog post is executable python code, it requires the gevent modules to run
+# pip as gevent, Debian as python3-gevent.
+
+import os
+import sys
+import gevent
+import gevent.socket as socket
+import gevent.ssl as ssl  # we must use gevent's ssl module here, see ln 186
+import signal
+from io import StringIO
+from email.utils import formatdate
+
+Fork = False
+# Welcome to Part 5.5 of this blog post series that might actually be a
+# book. Wherein we shall attempt to implement TLS correctly for this
+# our bespoke http implementation. Which if I did my research correctly.
+
+# Should look easy, but figureing out how not to shoot ourselves in
+# the foot took almost five hours of research.
+# A reminder please don't use this code in production.
+# If you want to fork it and make your own mistakes
+# after Part VI comes out be my guest, your own misfortune.
+
+# You should have also seen the prologue to the code.
+# which advises you to install mkcert, and tells you how to use it
+# to get a valid cert/keypair for use in this example.
+# If you didn't read it **READ IT NOW**
+
+
+CRLF = "\r\n"
+LF = "\n"
+
+
+class AccessDict(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Convert nested dicts to AccessDict
+        for key, value in self.items():
+            if isinstance(value, dict) and not isinstance(value, AccessDict):
+                self[key] = AccessDict(value)
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(
+                f"'{type(self).__name__}' object has no attribute '{key}'"
+            )
+
+    def __setattr__(self, key, value):
+        # Convert nested dicts to AccessDict
+        if isinstance(value, dict) and not isinstance(value, AccessDict):
+            value = AccessDict(value)
+        self[key] = value
+
+    def __delattr__(self, key):
+        try:
+            del self[key]
+        except KeyError:
+            raise AttributeError(
+                f"'{type(self).__name__}' object has no attribute '{key}'"
+            )
+
+
+# These are request and response objects similar to Go's standard
+# net/http package. Again this is still a toy implementation.
+# In a future part. I will go over how to make a more full featured
+# implementation. This is just making us ssl ready and laying the ground work
+class HttpRequest(AccessDict):
+    def __init__(self, method="GET", path="/", headers={}, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self["method"] = method
+        self["headers"] = headers
+        if "body" in kwargs:
+            self["body"] = StringIO(kwargs["body"])
+        self["path"] = path
+        if "Host" not in self["headers"]:
+            self["headers"]["Host"] = "localhost"
+
+    def read(self, seek):
+        return self["body"].read(seek)
+
+    def __str__(self):
+        buf = StringIO()
+        buf.write(f"{self.method} {self.path} HTTP/1.1")
+        for k, v in self["headers"].items():
+            buf.write(f"{k}: {v}" + CRLF)
+        buf.write(CRLF)
+        buf.write(self["body"].getvalue() + CRLF)
+        return buf.getvalue() + CRLF
+
+
+class HttpResponse(AccessDict):
+    def __init__(self, status="400", headers={}, body="goodbye\r\n", *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self["status"] = status
+        self["headers"] = headers
+        self["body"] = StringIO()
+        # We Must have date and host headers set correctly to use tls
+        # so we unconditionally set them here
+        if "host" not in kwargs:
+            self["headers"]["Host"] = "localhost"
+        else:
+            self["headers"]["Host"] = kwargs["host"]
+
+        self["headers"]["Date"] = formatdate(timeval=None, localtime=False, usegmt=True)
+        self["headers"]["Content-Type"] = "text/plain; charset=UTF-8"
+
+    def write(self, stuff):
+        return self.body.write(stuff)
+
+    # Foreshadowing (n): A literary device in which an author ...
+    def __str__(self):
+        buf = StringIO()
+        print(self.headers)
+        buf.write(f"HTTP/1.1 {self.status}" + CRLF)
+        length = len(self["body"].getvalue())
+        for k, v in self["headers"].items():
+            buf.write(f"{k}: {v}\r\n")
+        if "Content-Length" not in self["headers"]:
+            buf.write(f"Content-Length: {length}\r\n")
+        buf.write(CRLF)  # Per RFC 9112
+
+        buf.write(self["body"].getvalue() + CRLF)
+        return buf.getvalue() + CRLF
+
+
+RICKROLL_LYRICS = """
+
+We're no strangers to love
+You know the rules and so do I
+A full commitment's what I'm thinkin' of
+You wouldn't get this from any other guy
+
+I just wanna tell you how I'm feeling
+Gotta make you understand
+
+Never gonna give you up, never gonna let you down
+Never gonna run around and desert you
+Never gonna make you cry, never gonna say goodbye
+Never gonna tell a lie and hurt you
+
+We've known each other for so long
+Your heart's been aching, but you're too shy to say it
+Inside, we both know what's been going on
+We know the game and we're gonna play it
+
+And if you ask me how I'm feeling
+Don't tell me you're too blind to see
+
+Never gonna give you up, never gonna let you down
+Never gonna run around and desert you
+Never gonna make you cry, never gonna say goodbye
+Never gonna tell a lie and hurt you
+
+Never gonna give you up, never gonna let you down
+Never gonna run around and desert you
+Never gonna make you cry, never gonna say goodbye
+Never gonna tell a lie and hurt you
+
+"""
+head_response = HttpResponse()
+head_response.status = 200
+head_response.headers["Content-Length"] = 980
+head_response.write("")
+good_response = HttpResponse()
+good_response.status = 200
+good_response.headers["Last-Modified"] = "Mon, 27 July 1987 00:00 GMT"
+good_response.write(RICKROLL_LYRICS)
+
+error_response = HttpResponse()
+error_response.status = 405  # a 405 here is closer to RFC compliant
+error_response.write("Together forever and never to part Together forever we two")
+
+
+client_procs = []
+
+# commentary on ln CHANGEME
+ct_svr_proc = None
+ssl_svr_proc = None
+
+
+# You've Seen all this before, in the last Part 1. I will shorten commentary
+class NullDevice:
+    def write(self, s):
+        pass
+
+
+def hup_handle(sig, fr):
+    sys.exit()
+
+
+# We've been using gevent all along, but now it's time to say the quiet part
+# outloud. Gevent is an alternative concurrency module for python.
+# Trying to use gevent and standard python's stuff side by side
+# is a quick road to madness. I'll explain more in the Prequal series
+# For now it's enough to know the APIs are identical.
+
+# With that out of the way, we come to our first real decision,
+# that has security implications.
+# We could implement TLS in two ways. Method 1 we have the server.
+# listen on two different ports. As http does on 80/443
+# the other approach involves encrypting traffic on the port we
+# already use. The one port method may seem safer.
+# This was the route chosen by the gemini project.
+
+
+# But for teaching purposes the two port method works better.
+# so that's what we'll do.
+# This requires a couple of changes, in server handler.
+# first we change it's name, and make the corresponding change
+# in main, and we'll copy it almost verbatim, and make changes for TLS
+def cleartext_server_handler():
+    serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    serversock.bind(("", 1337))
+    serversock.listen(10)
+    while True:
+        client, addr = serversock.accept()
+        print(addr)
+        client_procs.append(gevent.spawn(client_handler, client))
+        gevent.sleep(0.25)
+
+    serversock.close()
+    return
+
+
+# TLS looks easy, but in practice almost every library for handling it
+# in any language you can name is... HOT GARBAGE. Python is not the
+# exception, but i find it does have less security foot guns.
+
+
+# if anything screws up it will crash out with an SSLError
+# it's error messages are cryptic.
+# But once you've done it properly it looks easy.
+# This took about four hours to debug, but i've got it finally
+def tls_server_handler():
+    # the context is sort of like a container for cryptographic settings
+    # we load the default context, which contains the best default
+    # settings as reviewed, by the python security people.abs
+    # this avoids a lot of foot guns
+
+    # Note here that the server/client is reversed
+    # Because we are a server we need the context
+    # for clients.
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+
+    # Remember what i said about best default and not shooting
+    # ourselves in the foot. Well Firefox, does a stupid thing
+    # during TLS handshake and declares it supports. SSLv3
+    # Which has been considered hopelessly broken since 2014.
+    # Mozilla's own security people even said so at the time.
+    # See References. Anyway Setting the minimum and maximum
+    # explicitly to TLSv1.2/1.3 avoids this wrongness.
+    # so we do it
+    ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+    ctx.maximum_version = ssl.TLSVersion.TLSv1_3
+
+    serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    serversock.bind(("", 1972))  # we choose another port number, 1972 foreshadows
+    # a bit
+
+    # Next we load our key and cert
+    # I'm assuming you used the mkcert method
+    # if you used the self signed method just place your cert/key
+    # See comment on ln 277
+    ctx.load_cert_chain(certfile="cert.pem", keyfile="key.pem")
+
+    serversock.listen(10)
+
+    while True:
+        client, addr = serversock.accept()
+
+        print(addr)
+        try:
+            secure = ctx.wrap_socket(client, server_side=True)
+            # last step in the process is to wrap the client socket in
+            # TLS. The SSLContext does this for us.
+            # but you must pass server_side=True to avoid silly defaults
+            client_procs.append(gevent.spawn(client_handler, secure))
+            gevent.sleep(0.25)
+        # If anything goes wrong here the ssl.SSLError is thrown
+        # i was originally going to leave it to crash, but
+        # the browser behavior on pki errors also causes this
+        # exception. So we will log the error and continue.
+        except ssl.SSLError as e:
+            print(e)
+            gevent.sleep(0.25)
+            continue
+        finally:
+            gevent.sleep(0.25)
+
+    serversock.close()
+    return
+
+
+# One Change here see ln 325
+
+
+def client_handler(sock):
+    print("Client handler spawn")
+    junk_counter = 0
+    while True:
+        if junk_counter > 3:
+            sock.close()
+            return
+        data = sock.recv(4096)
+        dstring = data.decode("UTF-8")
+        if dstring.startswith("GET"):
+            break
+        elif dstring.startswith("HEAD"):
+            hr = str(head_response)
+            sock.send(hr.encode("utf-8"))
+            sock.shutdown(socket.SHUT_RDWR)
+            sock.close()
+            return
+        else:
+            error = str(error_response)
+            sock.send(error.encode("utf-8"))
+            junk_counter += 1
+
+    gevent.sleep(0.25)  # this is a somewhat magical value, see Part II
+    default = str(good_response)
+    sock.send(default.encode("utf-8"))
+    sock.shutdown(socket.SHUT_RDWR)  # we do a more graceful exit here by
+    # shutting down the socket, makes things faster for TLS
+    # may have an effect on client response time to but i didn't notice it.
+    sock.close()
+    return
+
+
+def daemon_main():
+    svr_proc = gevent.spawn(cleartext_server_handler)
+    ssl_svr_proc = gevent.spawn(tls_server_handler)
+    client_procs.append(svr_proc)
+    client_procs.append(ssl_svr_proc)
+    gevent.joinall(client_procs)
+    sys.exit(0)
+
+
+# so things will not fork while i'm debbuging
+if not Fork:
+    daemon_main()
+
+pid = os.fork()
+if pid:
+    os._exit(0)
+
+
+else:
+    os.setpgrp()
+    os.umask(0)
+
+    print(os.getpid())
+    sys.stdout = NullDevice()
+    sys.stderr = NullDevice()
+
+    signal.signal(signal.SIGHUP, hup_handle)
+    signal.signal(signal.SIGTERM, hup_handle)
+    daemon_main()
+
+
+# To recap we just did a bunch of work, for no user visible change
+# This is not a bad thing, often the first drafts of programs.
+# Will fit the requirements of the moment. But when the requirements
+# change the program must be adapted to fit.
+# This process of iteration and redesign,
+# is called "paying down technical debt", and it should be done whenever
+# possible.
+#
+# And we've just moved up to the second level of the 7 story mountain
+# Yay us.
+#
+# References
+# Robustness Principal (Devopedia): https://devopedia.org/postel-s-law
+# IETF RFC 9112 HTTP/1.1 https://datatracker.ietf.org/doc/html/rfc9112
+#