load overviews from cache

to be nice and avoid hammering the server during development

bundle.py

@@ -1,32 +0,0 @@
-import zipfile
-import glob
-import os
-
-
-# TODO: should we include .pyc files?
-# TODO: add urwid source into the repo somewhere
-
-files = {
-    '__main__.py': 'clients/urwid/main.py',
-    'nntp_client.py': 'clients/nntp_client.py',
-    'urwid': 'env/lib/python3.8/site-packages/urwid/*.py',
-}
-
-with open('bbj_demo', 'wb') as f:
-    f.write(b"#!/usr/bin/env python3\n")
-    with zipfile.ZipFile(f, 'w', compression=zipfile.ZIP_DEFLATED) as z:
-        z.comment = b'BBJ'
-        for name, source in files.items():
-            if '*' in source:
-                dirname = name
-                for path in sorted(glob.glob(source)):
-                    name = dirname + '/' + os.path.basename(path)
-                    z.write(path, name)
-            else:
-                z.write(source, name)
-try:
-    mask = os.umask(0)
-    os.umask(mask)
-except OSError:
-    mask = 0
-os.chmod(z.filename, 0o777&~mask)

clients/nntp_client.py

@@ -1,18 +1,15 @@
-from collections import namedtuple
 from urllib.error import URLError
 import urllib.request as url
-from hashlib import sha256
-import email.utils
 import nntplib
+from hashlib import sha256
 import time
 import json
 import ssl
-import re
+import email.utils
+from collections import namedtuple
 
 import os
 
-__all__ = ('BBJNews','URLError')
-
 class BBJNews(object):
     # this module isnt exactly complete. The below description claims
     # `all of its endpoints are mapped to native methods` though this
@@ -509,8 +506,35 @@ class BBJNews(object):
         #       :bytes - the number of bytes in the article
         #       :lines - the number of lines in the body (deprecated)
 
-        # see also: https://www.jwz.org/doc/threading.html
-        threads = _overviews_to_threads_fancy(overviews)
+        if False:
+            # build up a map of message references
+            # we use a disjoint-set data structure
+            # to find the root of each message
+            threadmap = {}
+            rank = {}
+            for num, ov in overviews:
+                msgid = nntplib.decode_header(ov['message-id'])
+                # RFC5536 suggests that whitespace should not occur inside
+                # a message id, which (if true) makes it pretty easy to split
+                # the list of message ids in the references header
+                refs = nntplib.decode_header(ov['references']).split()
+                for r in refs:
+                    threadmap[msgid] = r
+                    rank[msgid] = 1
+                    # TODO
+        else:
+            # make every message its own thread, for prototyping purposes
+            #t = {
+            #    'title': str,
+            #    'reply_count': int, # does this include the OP?
+            #    'pinned': bool,
+            #    'thread_id': uuid
+            #    'author': user_uuid,
+            #    'created': time,
+            #    'last_mod': time,
+            #    'last_author': user_uuid,
+            #}
+            threads = _overview_to_threads(overviews)
 
         # make usermap
         usermap = {}
@@ -521,13 +545,12 @@ class BBJNews(object):
             addr = _parse_single_address(userid)
             usermap[userid] = {
                 'user_id': userid,
-                'user_name': addr.name or addr.user,
+                'user_name': addr.name,
                 'address': addr.address,
                 'color': colorhash(userid),
                 'is_admin': False, # TODO: LIST MODERATORS?
             }
 
-        threads.sort(key=lambda x: x['last_mod'], reverse=True)
         return threads, usermap
 
 
@@ -542,8 +565,7 @@ class BBJNews(object):
               print(usermap[author_id]["user_name"])
               print(message["body"])
         """
-        m = self.fake_message('oops...')
-        return {"title":"", "messages":[m], "author":m['author']}, {m['author']: self.user}
+        return {}, {}
 
         response = self("thread_load",
             format=format, thread_id=thread_id, op_only=op_only)
@@ -593,14 +615,14 @@ class BBJNews(object):
         }
 
 
+    # unused
     def format_message(self, body, format="sequential"):
         """
         Send `body` to the server to be formatted according to `format`,
         defaulting to the sequential parser. Returns the body object.
         """
-        return [[(None, body)]]
-        #response = self("format_message", body=body, format=format)
-        #return response["data"]
+        response = self("format_message", body=body, format=format)
+        return response["data"]
 
     # unsupported
     def message_delete(self, thread_id, post_id):
@@ -725,78 +747,8 @@ class BBJNews(object):
             "messages": response["data"]["messages"]
         }
 
-def _overviews_to_threads_fancy(overviews):
-    # build up a map of message references
-    # we use a disjoint-set data structure
-    # to find the root of each message
-    threadmap = {}
-    def find(id):
-        parent = threadmap.setdefault(id, id)
-        if parent == id:
-            return id
-        root = find(parent)
-        if root != parent:
-            threadmap[id] = root
-        return root
-
-    messages = {}
-    for num, ov in overviews:
-        try:
-            msgid = nntplib.decode_header(ov['message-id']).strip()
-            refs = _parse_message_ids(nntplib.decode_header(ov['references']))
-        except ValueError:
-            continue
-
-        messages[msgid] = (num, msgid, ov)
-        for r in refs:
-            threadmap[find(msgid)] = find(r)
-
-    thread_messages = {}
-    for id in messages:
-        root = find(id)
-        l = thread_messages.setdefault(root, [])
-        l.append(messages[id])
-
-    threads = []
-    for id, messages in thread_messages.items():
-        messages.sort(key=lambda x: x[0])
-        first = messages[0][2]
-        last = messages[-1][2]
-        try:
-            d = nntplib.decode_header(first['date'])
-            d = email.utils.mktime_tz(email.utils.parsedate_tz(d))
-            d2 = nntplib.decode_header(last['date'])
-            d2 = email.utils.mktime_tz(email.utils.parsedate_tz(d2))
-            t = {
-                'pinned': False,
-                'title': nntplib.decode_header(first['subject']),
-                'reply_count': len(messages),
-                'thread_id': nntplib.decode_header(first['message-id']),
-                'author': nntplib.decode_header(first['from']),
-                'created': d,
-                'last_author': nntplib.decode_header(last['from']),
-                'last_mod': d2,
-            }
-        except (ValueError, KeyError, IndexError):
-            continue
-        else:
-            threads.append(t)
-
-    return threads
-
 
 def _overview_to_threads(overviews):
-    # make every message its own thread, for prototyping purposes
-    #t = {
-    #    'title': str,
-    #    'reply_count': int, # does this include the OP?
-    #    'pinned': bool,
-    #    'thread_id': uuid
-    #    'author': user_uuid,
-    #    'created': time,
-    #    'last_mod': time,
-    #    'last_author': user_uuid,
-    #}
     threads = []
     for num, ov in overviews:
         try:
@@ -830,20 +782,7 @@ def _test_overview_to_threads():
         print(t)
 
 
-_atext = r"[a-zA-Z0-9!#$%&'\*\+\-/=?^_`{|}~]" # RFC 5322 §3.2.3
-_dotatext = r"%s+(?:\.%s+)*" % (_atext, _atext)
-_mdtext = r"\[[!-=\?-Z^-~]\]"
-_msg_id_re = re.compile(r'<%s@(?:%s|%s)>' % (_dotatext, _dotatext, _mdtext)) # RFC 5536 §3.1.3
-
-def _parse_message_ids(s):
-    """parses a list of message ids separated by junk"""
-    return _msg_id_re.findall(s)
-
-class Address(namedtuple('Address', 'name, address')):
-    @property
-    def user(self):
-        user, _, _ = self.address.partition("@")
-        return user
+Address = namedtuple('Address', 'name, address')
 
 def _parse_single_address(value):
     # the email.headerregistry api is truly bizarre

clients/urwid/main.py

@@ -796,7 +796,7 @@ class App(object):
         """
         if self.mode == "thread":
             # mark the current position in this thread before going back to the index
-            self.mark()
+            mark()
 
         self.body.attr_map = {None: "default"}
         self.mode = "index"
@@ -862,7 +862,7 @@ class App(object):
             self.walker += self.make_message_body(message)
         self.set_default_header()
         self.set_default_footer()
-        self.goto_mark(thread_id)
+        self.goto_post(mark(thread_id))
 
 
     def toggle_client_pin(self):
@@ -986,10 +986,10 @@ class App(object):
                 self.last_index_pos = self.get_focus_post(True).thread["thread_id"]
             self.index()
         else:
-            self.mark()
+            mark()
             thread = self.thread["thread_id"]
             self.thread_load(None, thread)
-            self.goto_mark(thread)
+            self.goto_post(mark(thread))
         self.temp_footer_message("Refreshed content!", 1)
 
 
@@ -1024,31 +1024,10 @@ class App(object):
                 width=30, height=6)
 
         else:
-            self.mark()
+            mark()
             self.index()
 
 
-    def mark(self, thread_id=None):
-        if self.mode != "thread":
-            return
-
-        if thread_id is None:
-            thread_id = self.thread['thread_id']
-        pos = self.get_focus_post()
-        mark(thread_id, pos, default=0)
-        return pos
-
-
-    def goto_mark(self, thread_id=None):
-        if self.mode != "thread":
-            return
-
-        if thread_id is None:
-            thread_id = self.thread['thread_id']
-        pos = mark(thread_id, default=0)
-        self.goto_post(pos)
-
-
     def get_focus_post(self, return_widget=False):
         pos = self.box.get_focus_path()[0]
         if self.mode == "thread":
@@ -2525,16 +2504,11 @@ def bbjrc(mode, **params):
     return values
 
 
-def mark(key, value=None, default=None):
+def mark(directive=True):
     """
-    Sets a value in the markfile and returns the old value (or default).
+    Set and retrieve positional marks for threads.
     This uses a seperate file from the preferences
     to keep it free from clutter.
-
-    The key must be a string, and the value must be
-    json-encodable. If value isn't provided (or is None)
-    then this doesn't set anything and it is only a
-    read operation.
     """
     try:
         with open(markpath, "r") as _in:
@@ -2542,14 +2516,19 @@ def mark(key, value=None, default=None):
     except FileNotFoundError:
         values = {}
 
-    old = values.get(key, default)
-
-    if value is not None and value != old:
-        values[key] = value
+    if directive == True and app.mode == "thread":
+        pos = app.get_focus_post()
+        values[app.thread["thread_id"]] = pos
         with open(markpath, "w") as _out:
             json.dump(values, _out)
+        return pos
+
+    elif isinstance(directive, str):
+        try:
+            return values[directive]
+        except KeyError:
+            return 0
 
-    return old
 
 def load_client_pins():
     """