update bundle.py for nntp

to be nice and avoid hammering the server during development

bundle.py

@@ -0,0 +1,32 @@
+import zipfile
+import glob
+import os
+
+
+# TODO: should we include .pyc files?
+# TODO: add urwid source into the repo somewhere
+
+files = {
+    '__main__.py': 'clients/urwid/main.py',
+    'nntp_client.py': 'clients/nntp_client.py',
+    'urwid': 'env/lib/python3.8/site-packages/urwid/*.py',
+}
+
+with open('bbj_demo', 'wb') as f:
+    f.write(b"#!/usr/bin/env python3\n")
+    with zipfile.ZipFile(f, 'w', compression=zipfile.ZIP_DEFLATED) as z:
+        z.comment = b'BBJ'
+        for name, source in files.items():
+            if '*' in source:
+                dirname = name
+                for path in sorted(glob.glob(source)):
+                    name = dirname + '/' + os.path.basename(path)
+                    z.write(path, name)
+            else:
+                z.write(source, name)
+try:
+    mask = os.umask(0)
+    os.umask(mask)
+except OSError:
+    mask = 0
+os.chmod(z.filename, 0o777&~mask)

clients/nntp_client.py

@@ -1,15 +1,18 @@
+from collections import namedtuple
 from urllib.error import URLError
 import urllib.request as url
-import nntplib
 from hashlib import sha256
+import email.utils
+import nntplib
 import time
 import json
 import ssl
-import email.utils
-from collections import namedtuple
+import re
 
 import os
 
+__all__ = ('BBJNews','URLError')
+
 class BBJNews(object):
     # this module isnt exactly complete. The below description claims
     # `all of its endpoints are mapped to native methods` though this
@@ -506,35 +509,8 @@ class BBJNews(object):
         #       :bytes - the number of bytes in the article
         #       :lines - the number of lines in the body (deprecated)
 
-        if False:
-            # build up a map of message references
-            # we use a disjoint-set data structure
-            # to find the root of each message
-            threadmap = {}
-            rank = {}
-            for num, ov in overviews:
-                msgid = nntplib.decode_header(ov['message-id'])
-                # RFC5536 suggests that whitespace should not occur inside
-                # a message id, which (if true) makes it pretty easy to split
-                # the list of message ids in the references header
-                refs = nntplib.decode_header(ov['references']).split()
-                for r in refs:
-                    threadmap[msgid] = r
-                    rank[msgid] = 1
-                    # TODO
-        else:
-            # make every message its own thread, for prototyping purposes
-            #t = {
-            #    'title': str,
-            #    'reply_count': int, # does this include the OP?
-            #    'pinned': bool,
-            #    'thread_id': uuid
-            #    'author': user_uuid,
-            #    'created': time,
-            #    'last_mod': time,
-            #    'last_author': user_uuid,
-            #}
-            threads = _overview_to_threads(overviews)
+        # see also: https://www.jwz.org/doc/threading.html
+        threads = _overviews_to_threads_fancy(overviews)
 
         # make usermap
         usermap = {}
@@ -545,12 +521,13 @@ class BBJNews(object):
             addr = _parse_single_address(userid)
             usermap[userid] = {
                 'user_id': userid,
-                'user_name': addr.name,
+                'user_name': addr.name or addr.user,
                 'address': addr.address,
                 'color': colorhash(userid),
                 'is_admin': False, # TODO: LIST MODERATORS?
             }
 
+        threads.sort(key=lambda x: x['last_mod'], reverse=True)
         return threads, usermap
 
 
@@ -565,7 +542,8 @@ class BBJNews(object):
               print(usermap[author_id]["user_name"])
               print(message["body"])
         """
-        return {}, {}
+        m = self.fake_message('oops...')
+        return {"title":"", "messages":[m], "author":m['author']}, {m['author']: self.user}
 
         response = self("thread_load",
             format=format, thread_id=thread_id, op_only=op_only)
@@ -615,14 +593,14 @@ class BBJNews(object):
         }
 
 
-    # unused
     def format_message(self, body, format="sequential"):
         """
         Send `body` to the server to be formatted according to `format`,
         defaulting to the sequential parser. Returns the body object.
         """
-        response = self("format_message", body=body, format=format)
-        return response["data"]
+        return [[(None, body)]]
+        #response = self("format_message", body=body, format=format)
+        #return response["data"]
 
     # unsupported
     def message_delete(self, thread_id, post_id):
@@ -747,8 +725,78 @@ class BBJNews(object):
             "messages": response["data"]["messages"]
         }
 
+def _overviews_to_threads_fancy(overviews):
+    # build up a map of message references
+    # we use a disjoint-set data structure
+    # to find the root of each message
+    threadmap = {}
+    def find(id):
+        parent = threadmap.setdefault(id, id)
+        if parent == id:
+            return id
+        root = find(parent)
+        if root != parent:
+            threadmap[id] = root
+        return root
+
+    messages = {}
+    for num, ov in overviews:
+        try:
+            msgid = nntplib.decode_header(ov['message-id']).strip()
+            refs = _parse_message_ids(nntplib.decode_header(ov['references']))
+        except ValueError:
+            continue
+
+        messages[msgid] = (num, msgid, ov)
+        for r in refs:
+            threadmap[find(msgid)] = find(r)
+
+    thread_messages = {}
+    for id in messages:
+        root = find(id)
+        l = thread_messages.setdefault(root, [])
+        l.append(messages[id])
+
+    threads = []
+    for id, messages in thread_messages.items():
+        messages.sort(key=lambda x: x[0])
+        first = messages[0][2]
+        last = messages[-1][2]
+        try:
+            d = nntplib.decode_header(first['date'])
+            d = email.utils.mktime_tz(email.utils.parsedate_tz(d))
+            d2 = nntplib.decode_header(last['date'])
+            d2 = email.utils.mktime_tz(email.utils.parsedate_tz(d2))
+            t = {
+                'pinned': False,
+                'title': nntplib.decode_header(first['subject']),
+                'reply_count': len(messages),
+                'thread_id': nntplib.decode_header(first['message-id']),
+                'author': nntplib.decode_header(first['from']),
+                'created': d,
+                'last_author': nntplib.decode_header(last['from']),
+                'last_mod': d2,
+            }
+        except (ValueError, KeyError, IndexError):
+            continue
+        else:
+            threads.append(t)
+
+    return threads
+
 
 def _overview_to_threads(overviews):
+    # make every message its own thread, for prototyping purposes
+    #t = {
+    #    'title': str,
+    #    'reply_count': int, # does this include the OP?
+    #    'pinned': bool,
+    #    'thread_id': uuid
+    #    'author': user_uuid,
+    #    'created': time,
+    #    'last_mod': time,
+    #    'last_author': user_uuid,
+    #}
     threads = []
     for num, ov in overviews:
         try:
@@ -782,7 +830,20 @@ def _test_overview_to_threads():
         print(t)
 
 
-Address = namedtuple('Address', 'name, address')
+_atext = r"[a-zA-Z0-9!#$%&'\*\+\-/=?^_`{|}~]" # RFC 5322 §3.2.3
+_dotatext = r"%s+(?:\.%s+)*" % (_atext, _atext)
+_mdtext = r"\[[!-=\?-Z^-~]\]"
+_msg_id_re = re.compile(r'<%s@(?:%s|%s)>' % (_dotatext, _dotatext, _mdtext)) # RFC 5536 §3.1.3
+
+def _parse_message_ids(s):
+    """parses a list of message ids separated by junk"""
+    return _msg_id_re.findall(s)
+
+class Address(namedtuple('Address', 'name, address')):
+    @property
+    def user(self):
+        user, _, _ = self.address.partition("@")
+        return user
 
 def _parse_single_address(value):
     # the email.headerregistry api is truly bizarre

clients/urwid/main.py

@@ -796,7 +796,7 @@ class App(object):
         """
         if self.mode == "thread":
             # mark the current position in this thread before going back to the index
-            mark()
+            self.mark()
 
         self.body.attr_map = {None: "default"}
         self.mode = "index"
@@ -862,7 +862,7 @@ class App(object):
             self.walker += self.make_message_body(message)
         self.set_default_header()
         self.set_default_footer()
-        self.goto_post(mark(thread_id))
+        self.goto_mark(thread_id)
 
 
     def toggle_client_pin(self):
@@ -986,10 +986,10 @@ class App(object):
                 self.last_index_pos = self.get_focus_post(True).thread["thread_id"]
             self.index()
         else:
-            mark()
+            self.mark()
             thread = self.thread["thread_id"]
             self.thread_load(None, thread)
-            self.goto_post(mark(thread))
+            self.goto_mark(thread)
         self.temp_footer_message("Refreshed content!", 1)
 
 
@@ -1024,10 +1024,31 @@ class App(object):
                 width=30, height=6)
 
         else:
-            mark()
+            self.mark()
             self.index()
 
 
+    def mark(self, thread_id=None):
+        if self.mode != "thread":
+            return
+
+        if thread_id is None:
+            thread_id = self.thread['thread_id']
+        pos = self.get_focus_post()
+        mark(thread_id, pos, default=0)
+        return pos
+
+
+    def goto_mark(self, thread_id=None):
+        if self.mode != "thread":
+            return
+
+        if thread_id is None:
+            thread_id = self.thread['thread_id']
+        pos = mark(thread_id, default=0)
+        self.goto_post(pos)
+
+
     def get_focus_post(self, return_widget=False):
         pos = self.box.get_focus_path()[0]
         if self.mode == "thread":
@@ -2504,11 +2525,16 @@ def bbjrc(mode, **params):
     return values
 
 
-def mark(directive=True):
+def mark(key, value=None, default=None):
     """
-    Set and retrieve positional marks for threads.
+    Sets a value in the markfile and returns the old value (or default).
     This uses a seperate file from the preferences
     to keep it free from clutter.
+
+    The key must be a string, and the value must be
+    json-encodable. If value isn't provided (or is None)
+    then this doesn't set anything and it is only a
+    read operation.
     """
     try:
         with open(markpath, "r") as _in:
@@ -2516,19 +2542,14 @@ def mark(directive=True):
     except FileNotFoundError:
         values = {}
 
-    if directive == True and app.mode == "thread":
-        pos = app.get_focus_post()
-        values[app.thread["thread_id"]] = pos
+    old = values.get(key, default)
+
+    if value is not None and value != old:
+        values[key] = value
         with open(markpath, "w") as _out:
             json.dump(values, _out)
-        return pos
-
-    elif isinstance(directive, str):
-        try:
-            return values[directive]
-        except KeyError:
-            return 0
 
+    return old
 
 def load_client_pins():
     """