tildetown
/
town
3
2
Fork 1
Code Issues Pull Requests Releases Wiki Activity

move stats into its own library

pull/1/head
vilmibm 2023-02-28 23:17:59 +00:00
parent 5a41d99ff9
commit 9de98bf2ab
2 changed files with 341 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

337
cmd/stats/main.go
View File

@ -3,351 +3,20 @@
// License: GPLv3+ // License: GPLv3+
// TDP is defined at http://protocol.club/~datagrok/beta-wiki/tdp.html // TDP is defined at http://protocol.club/~datagrok/beta-wiki/tdp.html
// It is a JSON structure of the form:
// Usage: stats > /var/www/html/tilde.json // Usage: stats > /var/www/html/tilde.json
package main package main
import ( import (
"bufio"
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"log" "log"
"os"
"os/exec" "git.tilde.town/tildetown/town/stats"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
) )
const defaultIndexPath = "/etc/skel/public_html/index.html"
const description = `an intentional digital community for creating and sharing
works of art, peer education, and technological anachronism. we are
non-commercial, donation supported, and committed to rejecting false
technological progress in favor of empathy and sustainable computing.`
type newsEntry struct {
Title string `json:"title"` // Title of entry
Pubdate string `json:"pubdate"` // Human readable date
Content string `json:"content"` // HTML of entry
}
type user struct {
Username string `json:"username"` // Username of user
PageTitle string `json:"title"` // Title of user's HTML page, if they have one
Mtime int64 `json:"mtime"` // Timestamp representing the last time a user's index.html was modified
// Town additions
DefaultPage bool `json:"default"` // Whether or not user has updated their default index.html
}
type tildeData struct {
Name string `json:"name"` // Name of the server
URL string `json:"url"` // URL of the server's homepage
SignupURL string `json:"signup_url"` // URL for server's signup page
WantUsers bool `json:"want_users"` // Whether or not new users are being accepted
AdminEmail string `json:"admin_email"` // Email for server admin
Description string `json:"description"` // Description of server
UserCount int `json:"user_count"` // Total number of users on server sorted by last activity time
Users []*user `json:"users"`
// Town Additions
LiveUserCount int `json:"live_user_count"` // Users who have changed their index.html
ActiveUserCount int `json:"active_user_count"` // Users with an active session
ActiveUsers []string `json:"active_users"` // Usernames of logged in users
GeneratedAt string `json:"generated_at"` // When this was generated in '%Y-%m-%d %H:%M:%S' format
GeneratedAtSec int64 `json:"generated_at_sec"` // When this was generated in seconds since epoch
Uptime string `json:"uptime"` // output of `uptime -p`
News []newsEntry `json:"news"` // Collection of town news entries
}
func getEnvDefault(key, def string) string {
result := os.Getenv(key)
if result == "" {
result = def
}
return result
}
func homesDir() string { return getEnvDefault("HOMES_DIR", "/home") }
func getNews() (entries []newsEntry, err error) {
inMeta := true
inContent := false
current := newsEntry{}
blankLineRe := regexp.MustCompile(`^ *\n$`)
newsPath := getEnvDefault("NEWS_PATH", "/town/news.posts")
newsFile, err := os.Open(newsPath)
if err != nil {
return entries, fmt.Errorf("unable to read news file: %s", err)
}
defer newsFile.Close()
scanner := bufio.NewScanner(newsFile)
for scanner.Scan() {
newsLine := scanner.Text()
if strings.HasPrefix(newsLine, "#") || newsLine == "" || blankLineRe.FindStringIndex(newsLine) != nil {
continue
} else if strings.HasPrefix(newsLine, "--") {
entries = append(entries, current)
current = newsEntry{}
inMeta = true
inContent = false
} else if inMeta {
kv := strings.SplitN(newsLine, ":", 2)
if kv[0] == "pubdate" {
current.Pubdate = strings.TrimSpace(kv[1])
} else if kv[0] == "title" {
current.Title = strings.TrimSpace(kv[1])
} else {
log.Printf("ignoring unknown metadata in news entry: %v\n", newsLine)
}
if current.Pubdate != "" && current.Title != "" {
inMeta = false
inContent = true
}
} else if inContent {
current.Content += fmt.Sprintf("\n%v", strings.TrimSpace(newsLine))
} else {
panic("news post parsing should never reach this point")
}
}
return entries, nil
}
func indexPathFor(username string) (string, error) {
potentialPaths := []string{"index.html", "index.htm"}
indexPath := ""
errs := []error{}
for _, p := range potentialPaths {
fullPath := path.Join(homesDir(), username, "public_html", p)
_, staterr := os.Stat(fullPath)
if staterr != nil {
errs = append(errs, staterr)
} else {
indexPath = fullPath
break
}
}
if indexPath == "" {
return "", fmt.Errorf("failed to locate index file for %v; tried %v; encountered errors: %v", username, potentialPaths, errs)
}
return indexPath, nil
}
func pageTitleFor(username string) string {
pageTitleRe := regexp.MustCompile(`<title[^>]*>(.*)</title>`)
indexPath, err := indexPathFor(username)
if err != nil {
return ""
}
content, err := ioutil.ReadFile(indexPath)
if err != nil {
return ""
}
titleMatch := pageTitleRe.FindStringSubmatch(string(content))
if len(titleMatch) < 2 {
return ""
}
return titleMatch[1]
}
func systemUsers() map[string]bool {
systemUsers := map[string]bool{
"ubuntu": true,
"ttadmin": true,
"root": true,
}
envSystemUsers := os.Getenv("SYSTEM_USERS")
if envSystemUsers != "" {
for _, username := range strings.Split(envSystemUsers, ",") {
systemUsers[username] = true
}
}
return systemUsers
}
func mtimeFor(username string) int64 {
path := path.Join(homesDir(), username, "public_html")
var maxMtime int64 = 0
_ = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if maxMtime < info.ModTime().Unix() {
maxMtime = info.ModTime().Unix()
}
return nil
})
return maxMtime
}
func detectDefaultPageFor(username string, defaultHTML []byte) bool {
indexPath, err := indexPathFor(username)
if err != nil {
return false
}
indexFile, err := os.Open(indexPath)
if err != nil {
return false
}
defer indexFile.Close()
indexHTML, err := ioutil.ReadAll(indexFile)
if err != nil {
return false
}
return bytes.Equal(indexHTML, defaultHTML)
}
func getDefaultHTML() ([]byte, error) {
indexPath := os.Getenv("DEFAULT_INDEX_PATH")
if indexPath == "" {
indexPath = defaultIndexPath
}
defaultIndexFile, err := os.Open(indexPath)
if err != nil {
return []byte{}, fmt.Errorf("could not open default index: %s", err)
}
defer defaultIndexFile.Close()
defaultIndexHTML, err := ioutil.ReadAll(defaultIndexFile)
if err != nil {
return []byte{}, fmt.Errorf("could not read default index: %s", err)
}
return defaultIndexHTML, nil
}
type byMtime []*user
func (x byMtime) Len() int { return len(x) }
func (x byMtime) Less(i, j int) bool { return x[i].Mtime > x[j].Mtime } // because we want DESC
func (x byMtime) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func getUsers() (users []*user, err error) {
// For the purposes of this program, we discover users via:
// - presence in /home/
// - absence in systemUsers list (sourced from source code and potentially augmented by an environment variable)
// We formally used passwd parsing. This is definitely more "correct" and I'm
// not opposed to going back to that; going back to parsing /home is mainly to
// get this new version going.
defaultIndexHTML, err := getDefaultHTML()
if err != nil {
return users, err
}
out, err := exec.Command("ls", homesDir()).Output()
if err != nil {
return users, fmt.Errorf("could not run ls: %s", err)
}
scanner := bufio.NewScanner(bytes.NewReader(out))
systemUsers := systemUsers()
for scanner.Scan() {
username := scanner.Text()
if systemUsers[username] {
continue
}
user := user{
Username: username,
PageTitle: pageTitleFor(username),
Mtime: mtimeFor(username),
DefaultPage: detectDefaultPageFor(username, defaultIndexHTML),
}
users = append(users, &user)
}
return users, nil
}
func liveUserCount(users []*user) int {
count := 0
for _, u := range users {
if !u.DefaultPage {
count++
}
}
return count
}
func activeUsers() ([]string, error) {
out, err := exec.Command("sh", "-c", "who | cut -d' ' -f1 | sort -u").Output()
if err != nil {
return nil, fmt.Errorf("failed to get active user count: %w", err)
}
scanner := bufio.NewScanner(bytes.NewReader(out))
usernames := []string{}
for scanner.Scan() {
usernames = append(usernames, strings.TrimSpace(scanner.Text()))
}
return usernames, nil
}
func getUptime() (string, error) {
out, err := exec.Command("uptime").Output()
if err != nil {
return "", fmt.Errorf("could not run uptime: %s", err)
}
return strings.TrimSpace(string(out)), nil
}
func tdp() (tildeData, error) {
users, err := getUsers()
if err != nil {
return tildeData{}, fmt.Errorf("could not get user list: %s", err)
}
activeUsernames, err := activeUsers()
if err != nil {
return tildeData{}, fmt.Errorf("could not count active users: %s", err)
}
news, err := getNews()
if err != nil {
return tildeData{}, fmt.Errorf("could not get news: %s", err)
}
uptime, err := getUptime()
if err != nil {
return tildeData{}, fmt.Errorf("could not determine uptime: %s", err)
}
sort.Sort(byMtime(users))
return tildeData{
Name: "tilde.town",
URL: "https://tilde.town",
SignupURL: "https://cgi.tilde.town/users/signup",
WantUsers: true,
AdminEmail: "root@tilde.town",
Description: description,
UserCount: len(users),
Users: users,
LiveUserCount: liveUserCount(users),
ActiveUserCount: len(activeUsernames),
ActiveUsers: activeUsernames,
Uptime: uptime,
News: news,
GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"),
GeneratedAtSec: time.Now().Unix(),
}, nil
}
func main() { func main() {
systemData, err := tdp() systemData, err := stats.Stats()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

338
stats/stats.go 100644
View File

@ -0,0 +1,338 @@
package stats
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
)
const defaultIndexPath = "/etc/skel/public_html/index.html"
const description = `an intentional digital community for creating and sharing
works of art, peer education, and technological anachronism. we are
non-commercial, donation supported, and committed to rejecting false
technological progress in favor of empathy and sustainable computing.`
type NewsEntry struct {
Title string `json:"title"` // Title of entry
Pubdate string `json:"pubdate"` // Human readable date
Content string `json:"content"` // HTML of entry
}
type User struct {
Username string `json:"username"` // Username of user
PageTitle string `json:"title"` // Title of user's HTML page, if they have one
Mtime int64 `json:"mtime"` // Timestamp representing the last time a user's index.html was modified
// Town additions
DefaultPage bool `json:"default"` // Whether or not user has updated their default index.html
}
type TildeData struct {
Name string `json:"name"` // Name of the server
URL string `json:"url"` // URL of the server's homepage
SignupURL string `json:"signup_url"` // URL for server's signup page
WantUsers bool `json:"want_users"` // Whether or not new users are being accepted
AdminEmail string `json:"admin_email"` // Email for server admin
Description string `json:"description"` // Description of server
UserCount int `json:"user_count"` // Total number of users on server sorted by last activity time
Users []*User `json:"users"`
// Town Additions
LiveUserCount int `json:"live_user_count"` // Users who have changed their index.html
ActiveUserCount int `json:"active_user_count"` // Users with an active session
ActiveUsers []string `json:"active_users"` // Usernames of logged in users
GeneratedAt string `json:"generated_at"` // When this was generated in '%Y-%m-%d %H:%M:%S' format
GeneratedAtSec int64 `json:"generated_at_sec"` // When this was generated in seconds since epoch
Uptime string `json:"uptime"` // output of `uptime -p`
News []NewsEntry `json:"news"` // Collection of town news entries
}
func getEnvDefault(key, def string) string {
result := os.Getenv(key)
if result == "" {
result = def
}
return result
}
func homesDir() string { return getEnvDefault("HOMES_DIR", "/home") }
func getNews() (entries []NewsEntry, err error) {
inMeta := true
inContent := false
current := NewsEntry{}
blankLineRe := regexp.MustCompile(`^ *\n$`)
newsPath := getEnvDefault("NEWS_PATH", "/town/news.posts")
newsFile, err := os.Open(newsPath)
if err != nil {
return entries, fmt.Errorf("unable to read news file: %s", err)
}
defer newsFile.Close()
scanner := bufio.NewScanner(newsFile)
for scanner.Scan() {
newsLine := scanner.Text()
if strings.HasPrefix(newsLine, "#") || newsLine == "" || blankLineRe.FindStringIndex(newsLine) != nil {
continue
} else if strings.HasPrefix(newsLine, "--") {
entries = append(entries, current)
current = NewsEntry{}
inMeta = true
inContent = false
} else if inMeta {
kv := strings.SplitN(newsLine, ":", 2)
if kv[0] == "pubdate" {
current.Pubdate = strings.TrimSpace(kv[1])
} else if kv[0] == "title" {
current.Title = strings.TrimSpace(kv[1])
} else {
log.Printf("ignoring unknown metadata in news entry: %v\n", newsLine)
}
if current.Pubdate != "" && current.Title != "" {
inMeta = false
inContent = true
}
} else if inContent {
current.Content += fmt.Sprintf("\n%v", strings.TrimSpace(newsLine))
} else {
panic("news post parsing should never reach this point")
}
}
return entries, nil
}
func indexPathFor(username string) (string, error) {
potentialPaths := []string{"index.html", "index.htm"}
indexPath := ""
errs := []error{}
for _, p := range potentialPaths {
fullPath := path.Join(homesDir(), username, "public_html", p)
_, staterr := os.Stat(fullPath)
if staterr != nil {
errs = append(errs, staterr)
} else {
indexPath = fullPath
break
}
}
if indexPath == "" {
return "", fmt.Errorf("failed to locate index file for %v; tried %v; encountered errors: %v", username, potentialPaths, errs)
}
return indexPath, nil
}
func pageTitleFor(username string) string {
pageTitleRe := regexp.MustCompile(`<title[^>]*>(.*)</title>`)
indexPath, err := indexPathFor(username)
if err != nil {
return ""
}
content, err := ioutil.ReadFile(indexPath)
if err != nil {
return ""
}
titleMatch := pageTitleRe.FindStringSubmatch(string(content))
if len(titleMatch) < 2 {
return ""
}
return titleMatch[1]
}
func systemUsers() map[string]bool {
systemUsers := map[string]bool{
"ubuntu": true,
"ttadmin": true,
"root": true,
}
envSystemUsers := os.Getenv("SYSTEM_USERS")
if envSystemUsers != "" {
for _, username := range strings.Split(envSystemUsers, ",") {
systemUsers[username] = true
}
}
return systemUsers
}
func mtimeFor(username string) int64 {
path := path.Join(homesDir(), username, "public_html")
var maxMtime int64 = 0
_ = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if maxMtime < info.ModTime().Unix() {
maxMtime = info.ModTime().Unix()
}
return nil
})
return maxMtime
}
func detectDefaultPageFor(username string, defaultHTML []byte) bool {
indexPath, err := indexPathFor(username)
if err != nil {
return false
}
indexFile, err := os.Open(indexPath)
if err != nil {
return false
}
defer indexFile.Close()
indexHTML, err := ioutil.ReadAll(indexFile)
if err != nil {
return false
}
return bytes.Equal(indexHTML, defaultHTML)
}
func getDefaultHTML() ([]byte, error) {
indexPath := os.Getenv("DEFAULT_INDEX_PATH")
if indexPath == "" {
indexPath = defaultIndexPath
}
defaultIndexFile, err := os.Open(indexPath)
if err != nil {
return []byte{}, fmt.Errorf("could not open default index: %s", err)
}
defer defaultIndexFile.Close()
defaultIndexHTML, err := ioutil.ReadAll(defaultIndexFile)
if err != nil {
return []byte{}, fmt.Errorf("could not read default index: %s", err)
}
return defaultIndexHTML, nil
}
type byMtime []*User
func (x byMtime) Len() int { return len(x) }
func (x byMtime) Less(i, j int) bool { return x[i].Mtime > x[j].Mtime } // because we want DESC
func (x byMtime) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func getUsers() (users []*User, err error) {
// For the purposes of this program, we discover users via:
// - presence in /home/
// - absence in systemUsers list (sourced from source code and potentially augmented by an environment variable)
// We formally used passwd parsing. This is definitely more "correct" and I'm
// not opposed to going back to that; going back to parsing /home is mainly to
// get this new version going.
defaultIndexHTML, err := getDefaultHTML()
if err != nil {
return users, err
}
out, err := exec.Command("ls", homesDir()).Output()
if err != nil {
return users, fmt.Errorf("could not run ls: %s", err)
}
scanner := bufio.NewScanner(bytes.NewReader(out))
systemUsers := systemUsers()
for scanner.Scan() {
username := scanner.Text()
if systemUsers[username] {
continue
}
user := User{
Username: username,
PageTitle: pageTitleFor(username),
Mtime: mtimeFor(username),
DefaultPage: detectDefaultPageFor(username, defaultIndexHTML),
}
users = append(users, &user)
}
return users, nil
}
func liveUserCount(users []*User) int {
count := 0
for _, u := range users {
if !u.DefaultPage {
count++
}
}
return count
}
func activeUsers() ([]string, error) {
out, err := exec.Command("sh", "-c", "who | cut -d' ' -f1 | sort -u").Output()
if err != nil {
return nil, fmt.Errorf("failed to get active user count: %w", err)
}
scanner := bufio.NewScanner(bytes.NewReader(out))
usernames := []string{}
for scanner.Scan() {
usernames = append(usernames, strings.TrimSpace(scanner.Text()))
}
return usernames, nil
}
func getUptime() (string, error) {
out, err := exec.Command("uptime").Output()
if err != nil {
return "", fmt.Errorf("could not run uptime: %s", err)
}
return strings.TrimSpace(string(out)), nil
}
func Stats() (TildeData, error) {
users, err := getUsers()
if err != nil {
return TildeData{}, fmt.Errorf("could not get user list: %s", err)
}
activeUsernames, err := activeUsers()
if err != nil {
return TildeData{}, fmt.Errorf("could not count active users: %s", err)
}
news, err := getNews()
if err != nil {
return TildeData{}, fmt.Errorf("could not get news: %s", err)
}
uptime, err := getUptime()
if err != nil {
return TildeData{}, fmt.Errorf("could not determine uptime: %s", err)
}
sort.Sort(byMtime(users))
return TildeData{
Name: "tilde.town",
URL: "https://tilde.town",
SignupURL: "https://cgi.tilde.town/users/signup",
WantUsers: true,
AdminEmail: "root@tilde.town",
Description: description,
UserCount: len(users),
Users: users,
LiveUserCount: liveUserCount(users),
ActiveUserCount: len(activeUsernames),
ActiveUsers: activeUsernames,
Uptime: uptime,
News: news,
GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"),
GeneratedAtSec: time.Now().Unix(),
}, nil
}