2017-04-22 04:29:55 +00:00
|
|
|
|
#!/usr/bin/env python3
|
2015-10-20 19:50:23 +00:00
|
|
|
|
|
2015-10-20 21:49:20 +00:00
|
|
|
|
# stats.py - tilde data in tilde data protocol format.
|
2015-10-20 19:50:23 +00:00
|
|
|
|
# Copyright 2015 Michael F. Lamb <http://datagrok.org>
|
|
|
|
|
# License: GPLv3+
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
Outputs JSON data conforming to "~dp (Tilde Description Protocol)" as defined
|
|
|
|
|
at: http://protocol.club/~datagrok/beta-wiki/tdp.html
|
|
|
|
|
|
|
|
|
|
It is a JSON structure of the form:
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
'name': (string) the name of the server.
|
|
|
|
|
'url': (string) the URL of the server.
|
|
|
|
|
'signup_url': (string) the URL of a page describing the process required to request an account on the server.
|
|
|
|
|
'want_users': (boolean) whether the server is currently accepting new user requests.
|
|
|
|
|
'admin_email': (string) the email address of the primary server administrator.
|
|
|
|
|
'description': (string) a free-form description for the server.
|
2015-10-20 21:49:20 +00:00
|
|
|
|
'users': [ (array) an array of users on the server, sorted by last activity time
|
2015-10-20 19:50:23 +00:00
|
|
|
|
{
|
|
|
|
|
'username': (string) the username of the user.
|
|
|
|
|
'title': (string) the HTML title of the user’s index.html page.
|
|
|
|
|
'mtime': (number) a timestamp representing the last time the user’s index.html was modified.
|
|
|
|
|
},
|
|
|
|
|
...
|
|
|
|
|
]
|
2015-10-20 21:49:20 +00:00
|
|
|
|
'user_count': (number) the number of users currently registered on the server.
|
2015-10-20 19:50:23 +00:00
|
|
|
|
}
|
|
|
|
|
|
2015-10-20 21:49:20 +00:00
|
|
|
|
We also overload this with some data we were using in the previous version of
|
|
|
|
|
stats.py, which is of the form:
|
2015-10-20 19:50:23 +00:00
|
|
|
|
|
|
|
|
|
{
|
2015-10-20 21:49:20 +00:00
|
|
|
|
'users': [ (array) of users on the server.
|
2015-10-20 19:50:23 +00:00
|
|
|
|
{
|
|
|
|
|
'default': (boolean) Is the user still using their unmodified default index.html?
|
|
|
|
|
'favicon': (string) a url to an image representing the user
|
|
|
|
|
},
|
|
|
|
|
...
|
|
|
|
|
]
|
2015-10-20 21:49:20 +00:00
|
|
|
|
'live_user_count': (number) count of live users (those who have changed their index.html)
|
2015-10-20 19:50:23 +00:00
|
|
|
|
'active_user_count': (number) count of currently logged in users
|
|
|
|
|
'generated_at': (string) the time this JSON was generated in '%Y-%m-%d %H:%M:%S' format.
|
|
|
|
|
'generated_at_msec': (number) the time this JSON was generated, in milliseconds since the epoch.
|
|
|
|
|
'uptime': (string) output of `uptime -p`
|
2016-09-05 10:42:43 +00:00
|
|
|
|
'news': collection of tilde.town news entries containing 'title', 'pubdate', and 'content', the latter being raw HTML
|
2015-10-20 19:50:23 +00:00
|
|
|
|
|
|
|
|
|
}
|
2015-10-20 21:51:50 +00:00
|
|
|
|
Usage: stats.py > /var/www/html/tilde.json
|
2015-10-20 19:50:23 +00:00
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
import datetime
|
2015-07-28 22:05:52 +00:00
|
|
|
|
import json
|
2015-10-20 19:50:23 +00:00
|
|
|
|
import os
|
|
|
|
|
import pwd
|
|
|
|
|
import re
|
2015-10-20 06:37:37 +00:00
|
|
|
|
import subprocess
|
2015-07-28 22:05:52 +00:00
|
|
|
|
|
2015-10-20 19:50:23 +00:00
|
|
|
|
SYSTEM_USERS = ['wiki', 'root', 'ubuntu', 'nate', 'nobody']
|
2015-10-20 06:37:37 +00:00
|
|
|
|
DEFAULT_HTML_FILENAME = "/etc/skel/public_html/index.html"
|
2016-09-05 10:42:43 +00:00
|
|
|
|
NEWS_PATH = '/home/vilmibm/news.posts'
|
|
|
|
|
blank_line_re = re.compile(r'\s*\n')
|
2015-10-20 19:50:23 +00:00
|
|
|
|
title_re = re.compile(r'<title[^>]*>(.*)</title>', re.DOTALL)
|
|
|
|
|
|
|
|
|
|
def active_user_count():
|
|
|
|
|
"""Return the count of unique usernames logged in."""
|
|
|
|
|
return len(set(line.split()[0] for line in
|
|
|
|
|
subprocess.check_output(
|
|
|
|
|
["who"], universal_newlines=True).splitlines()))
|
|
|
|
|
|
|
|
|
|
def get_title(indexhtml):
|
|
|
|
|
"""Given an html file, return the content of its <title>"""
|
|
|
|
|
with open(indexhtml, 'rt', errors='ignore') as fp:
|
|
|
|
|
title = title_re.search(fp.read())
|
|
|
|
|
if title:
|
|
|
|
|
return title.group(1)
|
|
|
|
|
|
|
|
|
|
def get_users():
|
|
|
|
|
"""Generate tuples of the form (username, homedir) for all normal
|
|
|
|
|
users on this system.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
return ((p.pw_name, p.pw_dir) for p in pwd.getpwall() if
|
|
|
|
|
p.pw_uid >= 1000 and
|
|
|
|
|
p.pw_shell != '/bin/false' and
|
|
|
|
|
p.pw_name not in SYSTEM_USERS)
|
|
|
|
|
|
|
|
|
|
def most_recent_within(path):
|
|
|
|
|
"""Return the most recent timestamp among all files within path, 3
|
|
|
|
|
levels deep.
|
|
|
|
|
"""
|
2015-10-20 20:46:59 +00:00
|
|
|
|
return max(modified_times(path, maxdepth=3))
|
2015-10-20 19:50:23 +00:00
|
|
|
|
|
|
|
|
|
def modified_times(path, maxdepth=None):
|
|
|
|
|
"""Walk the directories in path, generating timestamps for all
|
|
|
|
|
files.
|
|
|
|
|
"""
|
|
|
|
|
for root, dirs, files in os.walk(path):
|
|
|
|
|
if maxdepth and len(root[len(path):].split(os.sep)) == maxdepth:
|
|
|
|
|
dirs.clear()
|
|
|
|
|
for f in files:
|
|
|
|
|
try:
|
|
|
|
|
yield os.path.getmtime(os.path.join(root, f))
|
2015-10-20 20:46:59 +00:00
|
|
|
|
except (FileNotFoundError, PermissionError):
|
2015-10-20 19:50:23 +00:00
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def tdp_user(username, homedir):
|
|
|
|
|
"""Given a unix username, and their home directory, return a TDP format
|
|
|
|
|
dict with information about that user.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
public_html = os.path.join(homedir, 'public_html')
|
|
|
|
|
index_html = os.path.join(public_html, 'index.html')
|
|
|
|
|
if os.path.exists(index_html):
|
|
|
|
|
return {
|
|
|
|
|
'username': username,
|
|
|
|
|
'title': get_title(index_html),
|
|
|
|
|
'mtime': int(most_recent_within(public_html) * 1000),
|
|
|
|
|
# tilde.town extensions and backward compatibility
|
|
|
|
|
'favicon': 'TODO',
|
|
|
|
|
'default': subprocess.call(
|
|
|
|
|
['diff', '-q', DEFAULT_HTML_FILENAME, index_html],
|
|
|
|
|
stdout=subprocess.DEVNULL) == 0,
|
|
|
|
|
}
|
|
|
|
|
else:
|
|
|
|
|
return {
|
|
|
|
|
'username': username,
|
|
|
|
|
'default': False
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-05 10:42:43 +00:00
|
|
|
|
def parse_news(news_path):
|
|
|
|
|
"""Given a path to a .posts file, builds an returns a list of news entries with keys 'title', 'content', and 'pubdate'"""
|
|
|
|
|
metadata_keys = ['title', 'pubdate']
|
|
|
|
|
in_meta = True
|
|
|
|
|
in_content = False
|
2016-09-05 11:15:52 +00:00
|
|
|
|
current_entry = {'content':''}
|
2016-09-05 10:42:43 +00:00
|
|
|
|
entries = []
|
|
|
|
|
with open(news_path, 'r') as f:
|
2016-09-05 11:15:52 +00:00
|
|
|
|
line = 'not null'
|
2016-09-05 10:42:43 +00:00
|
|
|
|
while line:
|
2016-09-05 11:15:52 +00:00
|
|
|
|
line = f.readline()
|
2016-09-05 10:42:43 +00:00
|
|
|
|
if blank_line_re.match(line) or line.startswith('#'):
|
|
|
|
|
continue
|
2016-09-05 11:15:52 +00:00
|
|
|
|
elif line == '--\n':
|
|
|
|
|
entries.append(current_entry)
|
|
|
|
|
current_entry = {'content':''}
|
|
|
|
|
in_meta = True
|
|
|
|
|
in_content = False
|
|
|
|
|
elif in_meta:
|
2016-09-05 10:42:43 +00:00
|
|
|
|
key, value = line.split(':', 1)
|
2016-09-05 11:15:52 +00:00
|
|
|
|
current_entry[key] = value.rstrip().lstrip()
|
2016-09-05 19:59:29 +00:00
|
|
|
|
if set(metadata_keys).issubset(current_entry.keys()):
|
2016-09-05 10:42:43 +00:00
|
|
|
|
in_content = True
|
|
|
|
|
in_meta = False
|
2016-09-05 11:15:52 +00:00
|
|
|
|
elif in_content:
|
|
|
|
|
current_entry['content'] += "\n{}".format(line.lstrip().rstrip())
|
2016-09-05 10:42:43 +00:00
|
|
|
|
|
|
|
|
|
return entries
|
|
|
|
|
|
2015-10-20 19:50:23 +00:00
|
|
|
|
def tdp():
|
|
|
|
|
now = datetime.datetime.now()
|
2015-10-20 21:49:20 +00:00
|
|
|
|
users = sorted(
|
|
|
|
|
(tdp_user(u, h) for u, h in get_users()),
|
2017-04-17 05:30:09 +00:00
|
|
|
|
key=lambda x:x.get('mtime', 0),
|
2015-10-20 21:49:20 +00:00
|
|
|
|
reverse=True)
|
2015-10-20 19:50:23 +00:00
|
|
|
|
|
|
|
|
|
# TDP format data
|
|
|
|
|
data = {
|
|
|
|
|
'name': 'tilde.town',
|
2016-09-05 10:56:27 +00:00
|
|
|
|
'url': 'https://tilde.town',
|
2018-02-26 17:37:20 +00:00
|
|
|
|
'signup_url': 'https://cgi.tilde.town/users/signup',
|
2015-10-20 19:50:23 +00:00
|
|
|
|
'want_users': True,
|
2016-09-05 10:42:43 +00:00
|
|
|
|
'admin_email': 'nathanielksmith@gmail.com',
|
2015-10-20 19:50:23 +00:00
|
|
|
|
'description': " ".join(l.strip() for l in """
|
|
|
|
|
an intentional digital community for creating and sharing works of
|
|
|
|
|
art, educating peers, and technological anachronism. we are a
|
|
|
|
|
completely non-commercial, donation supported, and committed to
|
|
|
|
|
rejecting false technological progress in favor of empathy and
|
|
|
|
|
sustainable computing.
|
|
|
|
|
""".splitlines()),
|
|
|
|
|
'user_count': len(users),
|
|
|
|
|
'users': users,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# tilde.town extensions and backward compatibility
|
|
|
|
|
data.update({
|
|
|
|
|
'active_user_count': active_user_count(),
|
|
|
|
|
'generated_at': now.strftime('%Y-%m-%d %H:%M:%S'),
|
|
|
|
|
'generated_at_msec': int(now.timestamp() * 1000),
|
|
|
|
|
'uptime': subprocess.check_output(['uptime', '-p'], universal_newlines=True),
|
2015-10-20 21:10:26 +00:00
|
|
|
|
'live_user_count': sum(1 for x in data['users'] if not x['default']),
|
2016-09-05 10:42:43 +00:00
|
|
|
|
'news': parse_news(NEWS_PATH),
|
2015-10-20 19:50:23 +00:00
|
|
|
|
})
|
2015-07-28 22:05:52 +00:00
|
|
|
|
|
2015-07-28 22:17:13 +00:00
|
|
|
|
return data
|
2015-07-28 22:05:52 +00:00
|
|
|
|
|
2015-10-20 19:50:23 +00:00
|
|
|
|
def main():
|
|
|
|
|
print(json.dumps(tdp(), sort_keys=True, indent=2))
|
|
|
|
|
|
2015-07-28 22:05:52 +00:00
|
|
|
|
if __name__ == '__main__':
|
2015-10-20 19:50:23 +00:00
|
|
|
|
raise SystemExit(main())
|