misc/filesystem.py

#!/usr/bin/env python3
## This Blog post is executable python code, it requires the gevent modules to run
# pip as gevent, Debian as python3-gevent.

import os
import sys
import gevent
import gevent.socket as socket
import gevent.ssl as ssl  # we must use gevent's ssl module here, see ln 186
import signal
from io import StringIO, BytesIO
from email.utils import formatdate
import mimetypes
import pathlib

Fork = False
# Welcome to Part 5.5 of this blog post series that might actually be a
# book. Wherein we shall attempt to implement TLS correctly for this
# our bespoke http implementation. Which if I did my research correctly.

# Should look easy, but figureing out how not to shoot ourselves in
# the foot took almost five hours of research.
# A reminder please don't use this code in production.
# If you want to fork it and make your own mistakes
# after Part VI comes out be my guest, your own misfortune.

# You should have also seen the prologue to the code.
# which advises you to install mkcert, and tells you how to use it
# to get a valid cert/keypair for use in this example.
# If you didn't read it **READ IT NOW**


CRLF = "\r\n"
LF = "\n"


class AccessDict(dict):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Convert nested dicts to AccessDict
        for key, value in self.items():
            if isinstance(value, dict) and not isinstance(value, AccessDict):
                self[key] = AccessDict(value)

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{key}'"
            )

    def __setattr__(self, key, value):
        # Convert nested dicts to AccessDict
        if isinstance(value, dict) and not isinstance(value, AccessDict):
            value = AccessDict(value)
        self[key] = value

    def __delattr__(self, key):
        try:
            del self[key]
        except KeyError:
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{key}'"
            )


# These are request and response objects similar to Go's standard
# net/http package. Again this is still a toy implementation.
# In a future part. I will go over how to make a more full featured
# implementation. This is just making us ssl ready and laying the ground work
class HttpRequest(AccessDict):
    def __init__(self, method="GET", path="/", headers={}, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self["method"] = method
        self["headers"] = headers
        if "body" in kwargs:
            self["body"] = StringIO(kwargs["body"])
        self["path"] = path
        if "Host" not in self["headers"]:
            self["headers"]["Host"] = "localhost"

    def read(self, seek):
        return self["body"].read(seek)

    def __str__(self):
        buf = StringIO()
        buf.write(f"{self.method} {self.path} HTTP/1.1\r\n")
        for k, v in self["headers"].items():
            if k == "":
                continue
            buf.write(f"{k}: {v}" + CRLF)
        buf.write(CRLF)
        if "body" in self:
            buf.write(self["body"].getvalue() + CRLF)
        else:
            buf.write("" + CRLF)
        return buf.getvalue() + CRLF


class HttpResponse(AccessDict):
    def __init__(self, status="400", headers={}, body="goodbye\r\n", *args, **kwargs):
        super().__init__(*args, **kwargs)
        self["status"] = status
        self["headers"] = headers
        self["body"] = StringIO()
        # We Must have date and host headers set correctly to use tls
        # so we unconditionally set them here
        if "host" not in kwargs:
            self["headers"]["Host"] = "localhost"
        else:
            self["headers"]["Host"] = kwargs["host"]
        if "binary" in kwargs and kwargs["binary"]:
            self.binmode = True
            self["body"] = BytesIO()  # hopefully this will be enough
        else:
            self.binmode = False

        self["headers"]["Date"] = formatdate(timeval=None, localtime=False, usegmt=True)
        self["headers"]["Content-Type"] = "text/plain; charset=UTF-8"

    def write(self, stuff):
        if self.binmode and type(stuff) is str:
            return self.body.write(bytes(stuff, "utf-8"))
        return self.body.write(stuff)

    def binary_io(self):
        if not self.binmode:
            str(self)
        hbuf = StringIO()
        bbuf = BytesIO()
        hbuf.write(f"HTTP/1.1 {self.status}" + CRLF)
        length = len(self["body"].getvalue())
        for k, v in self["headers"].items():
            hbuf.write(f"{k}: {v}\r\n")
        if "Content-Length" not in self["headers"]:
            hbuf.write(f"Content-Length: {length}\r\n")
        hbuf.write(CRLF)  # Per RFC 9112
        bbuf.write(self.body.getvalue())
        return (hbuf.getvalue(), bbuf.getvalue())

    def __str__(self):
        buf = StringIO()
        buf.write(f"HTTP/1.1 {self.status}" + CRLF)
        length = len(self["body"].getvalue())
        for k, v in self["headers"].items():
            buf.write(f"{k}: {v}\r\n")
        if "Content-Length" not in self["headers"]:
            buf.write(f"Content-Length: {length}\r\n")
        buf.write(CRLF)  # Per RFC 9112
        if self.binmode:
            buf.write(self["body"].getvalue().decode("utf-8"))
            return buf.getvalue().encode("utf-8").decode("utf-8")
        else:
            buf.write(self["body"].getvalue())
            return buf.getvalue() + CRLF


WEBROOT = "htdocs/"


def fs_handler(request):
    membuf = None
    wr = HttpResponse(binary=True, body=b"")
    target = pathlib.Path(__file__).parent
    target /= WEBROOT
    target /= request.path[1:]
    print("AFTER PATHLIB: " + str(target))
    print(str(pathlib.Path.cwd()))
    if str(target).endswith("/"):
        nt = target / "index.html"
        target = nt
        print("GETINDEX: " + str(target))

    if os.access(str(target), os.R_OK):
        try:
            membuf = open(target, "rb").read()
        except IsADirectoryError:
            nt = target / "index.html"
            if os.path.exists(str(nt)):
                membuf = open(nt, "rb").read()
                wr.write(membuf)
                wr.status = 200
                cts = mimetypes.guess_type(nt)[0]
                wr.headers["Content-Type"] = cts
                return wr
            else:
                wr.status = 404
                wr.write("May the force be with you\r\n")
                return wr
        wr.write(membuf)
        wr.status = 200
        wr.headers["Content-Type"] = mimetypes.guess_type(target)[0]
        return wr
    else:
        wr.status = 404
        wr.write("these are not the droids your looking for\r\n")
        return wr


RICKROLL_LYRICS = """

We're no strangers to love
You know the rules and so do I
A full commitment's what I'm thinkin' of
You wouldn't get this from any other guy

I just wanna tell you how I'm feeling
Gotta make you understand

Never gonna give you up, never gonna let you down
Never gonna run around and desert you
Never gonna make you cry, never gonna say goodbye
Never gonna tell a lie and hurt you

We've known each other for so long
Your heart's been aching, but you're too shy to say it
Inside, we both know what's been going on
We know the game and we're gonna play it

And if you ask me how I'm feeling
Don't tell me you're too blind to see

Never gonna give you up, never gonna let you down
Never gonna run around and desert you
Never gonna make you cry, never gonna say goodbye
Never gonna tell a lie and hurt you

Never gonna give you up, never gonna let you down
Never gonna run around and desert you
Never gonna make you cry, never gonna say goodbye
Never gonna tell a lie and hurt you

"""
head_response = HttpResponse()
head_response.status = 200
# head_response.headers["Content-Length"] = 0
head_response.write("")
good_response = HttpResponse()
good_response.status = 200
good_response.headers["Last-Modified"] = "Mon, 27 July 1987 00:00 GMT"
good_response.write(RICKROLL_LYRICS)

error_response = HttpResponse()
error_response.status = 404  # we fudge the rfc a bit here
error_response.write("Together forever and never to part Together forever we two")


client_procs = []

# commentary on ln CHANGEME
ct_svr_proc = None
ssl_svr_proc = None


# One change here
class BadParseError(Exception):
    """Exception raised for custom error in the application."""

    def __init__(self, message, error_code):
        super().__init__(message)
        self.error_code = error_code
        self.message = message

    def __str__(self):
        return f"{self.message} (Error Code: {self.error_code})"


# this exception will be raised by the request parser should an invalid
# request come in.
class NullDevice:
    def write(self, s):
        pass


def hup_handle(sig, fr):
    sys.exit()


# We've been using gevent all along, but now it's time to say the quiet part
# outloud. Gevent is an alternative concurrency module for python.
# Trying to use gevent and standard python's stuff side by side
# is a quick road to madness. I'll explain more in the Prequal series
# For now it's enough to know the APIs are identical.

# With that out of the way, we come to our first real decision,
# that has security implications.
# We could implement TLS in two ways. Method 1 we have the server.
# listen on two different ports. As http does on 80/443
# the other approach involves encrypting traffic on the port we
# already use. The one port method may seem safer.
# This was the route chosen by the gemini project.


# But for teaching purposes the two port method works better.
# so that's what we'll do.
# This requires a couple of changes, in server handler.
# first we change it's name, and make the corresponding change
# in main, and we'll copy it almost verbatim, and make changes for TLS
def cleartext_server_handler():
    serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
    serversock.bind(("", 1337))
    serversock.listen(10)
    while True:
        client, addr = serversock.accept()
        client_procs.append(gevent.spawn(client_handler, client))
        gevent.sleep(0.25)

    serversock.close()
    return


# TLS looks easy, but in practice almost every library for handling it
# in any language you can name is... HOT GARBAGE. Python is not the
# exception, but i find it does have less security foot guns.


# if anything screws up it will crash out with an SSLError
# it's error messages are cryptic.
# But once you've done it properly it looks easy.
# This took about four hours to debug, but i've got it finally
def tls_server_handler():
    # the context is sort of like a container for cryptographic settings
    # we load the default context, which contains the best default
    # settings as reviewed, by the python security people.abs
    # this avoids a lot of foot guns

    # Note here that the server/client is reversed
    # Because we are a server we need the context
    # for clients.
    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)

    # Remember what i said about best default and not shooting
    # ourselves in the foot. Well Firefox/Gevent/Python, does a stupid thing
    # during TLS handshake one  and declares it supports. SSLv3
    # Which has been considered hopelessly broken since 2014.
    # Mozilla's own security people even said so at the time.
    # See References. Anyway Setting the minimum and maximum
    # explicitly to TLSv1.2/1.3 avoids this wrongness.
    # so we do it
    ctx.minimum_version = ssl.TLSVersion.TLSv1_2
    ctx.maximum_version = ssl.TLSVersion.TLSv1_3

    serversock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
    serversock.bind(("", 1972))  # we choose another port number, 1972 foreshadows
    # a bit

    # Next we load our key and cert
    # I'm assuming you used the mkcert method
    # if you used the self signed method just place your cert/key
    # See comment on ln 277
    ctx.load_cert_chain(certfile="cert.pem", keyfile="key.pem")

    serversock.listen(10)

    while True:
        client, addr = serversock.accept()

        try:
            secure = ctx.wrap_socket(client, server_side=True)
            # last step in the process is to wrap the client socket in
            # TLS. The SSLContext does this for us.
            # but you must pass server_side=True to avoid silly defaults
            client_procs.append(gevent.spawn(client_handler, secure))
            gevent.sleep(0.25)
        # If anything goes wrong here the ssl.SSLError is thrown
        # i was originally going to leave it to crash, but
        # the browser behavior on pki errors also causes this
        # exception. So we will log the error and continue.
        except ssl.SSLError as e:
            print(e)
            gevent.sleep(0.25)
            continue
        finally:
            gevent.sleep(0.25)

    serversock.close()
    return


def routing_table(request):
    if request is not None:
        if request.method == "GET" and request.path == "/rick":
            return good_response
        elif request.method == "HEAD" and request.path == "/rick":
            return head_response
        elif request.method == "HEAD":
            return head_response
        else:
            return fs_handler(request)
    else:
        custom_response = HttpResponse()
        custom_response.status = 500
        custom_response.write("Oh bother" + CRLF)
        return custom_response


class HttpParser:
    def __init__(self, raw_request):
        try:
            self.request_text = raw_request.decode("utf-8")
        except UnicodeDecodeError:
            raise BadParseError("UTF-8 decode failed", 400)

    def parse(self):
        result = HttpRequest()
        atoms = self.request_text.split(CRLF)
        reqinfo = atoms[0].strip(CRLF).split(" ")
        if len(reqinfo) < 3:
            raise BadParseError("I'm a little teapot short and stout", 418)
        if reqinfo[2] != "HTTP/1.1":
            raise BadParseError("Wrong HTTP Version", 400)
        # Now we parse the headers
        for h in atoms[1:]:
            if h != CRLF or h != "":
                spl_point = h.find(":")
                kv = []
                kv.append(h[0:spl_point])
                kv.append(h[spl_point + 1 :])
                if len(kv) == 2:
                    result.headers[kv[0]] = kv[1]
                else:
                    raise BadParseError("here is my handle", 418)
            else:
                break
        result.method = reqinfo[0]
        result.path = reqinfo[1]
        return result


def client_handler(sock):
    print("Client handler spawn")
    junk_counter = 0
    p = None
    server_response = HttpResponse()
    while True:
        if junk_counter > 3:
            server_response.status = 420
            server_response.body.write("Try Some Indica, it may help")

            break
        data = sock.recv(4096)
        try:
            p = HttpParser(data).parse()
            server_response = routing_table(p)
            break
        except BadParseError as e:
            server_response = HttpResponse()
            server_response.status = e.error_code
            server_response.body.write(e.message + CRLF)
            junk_counter += 1
            default = str(server_response)
            sock.send(default.encode("utf-8"))
            continue

    # gevent.sleep(0.25)  # this is a somewhat magical value, see Part II
    if server_response.binmode:
        headers, body = server_response.binary_io()
        sock.send(headers.encode("utf-8"))
        sock.send(body)
    else:
        default = str(server_response) + CRLF
        sock.send(default.encode("utf-8"))
    # sock.shutdown(socket.SHUT_RDWR)  # we do a more graceful exit here by
    # shutting down the socket, makes things faster for TLS
    # may have an effect on client response time to but i didn't notice it.
    sock.close()
    return


def daemon_main():
    svr_proc = gevent.spawn(cleartext_server_handler)
    ssl_svr_proc = gevent.spawn(tls_server_handler)
    client_procs.append(svr_proc)
    client_procs.append(ssl_svr_proc)
    gevent.joinall(client_procs)
    sys.exit(0)


# so things will not fork while i'm debbuging
if __name__ == "__main__":
    if not Fork:
        daemon_main()

    pid = os.fork()
    if pid:
        os._exit(0)

    else:
        os.setpgrp()
        os.umask(0)

        print(os.getpid())
        sys.stdout = NullDevice()
        sys.stderr = NullDevice()

    signal.signal(signal.SIGHUP, hup_handle)
    signal.signal(signal.SIGTERM, hup_handle)
    daemon_main()


# Now for the recap, today we learned how TLS works from the server's
# Perspective. We learned that python has secure defaults and you
# should use them unless there's a problem.

# We also learned that firefox may have a problem, and being explicit
# about the TLS versions we support fixed that issue.

# This problem is more likely a bug in gevent, if I had to guess

# Stay tuned for Part 6 where we will, finally break up
# with Rick Astley.
# Cheers

# References
# Python's SSL Module docs: https://docs.python.org/3/library/ssl.html#ssl.create_default_context
# End of SSLv3: https://blog.mozilla.org/security/2014/10/14/the-poodle-attack-and-the-end-of-ssl-3-0/