From 9de98bf2ab80bb007e42fb6c359e5c9f49c8f103 Mon Sep 17 00:00:00 2001 From: vilmibm Date: Tue, 28 Feb 2023 23:17:59 +0000 Subject: [PATCH] move stats into its own library --- cmd/stats/main.go | 337 +-------------------------------------------- stats/stats.go | 338 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 341 insertions(+), 334 deletions(-) create mode 100644 stats/stats.go diff --git a/cmd/stats/main.go b/cmd/stats/main.go index d522b75..cc61fcd 100644 --- a/cmd/stats/main.go +++ b/cmd/stats/main.go @@ -3,351 +3,20 @@ // License: GPLv3+ // TDP is defined at http://protocol.club/~datagrok/beta-wiki/tdp.html -// It is a JSON structure of the form: - // Usage: stats > /var/www/html/tilde.json package main import ( - "bufio" - "bytes" "encoding/json" "fmt" - "io/ioutil" "log" - "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "sort" - "strings" - "time" + + "git.tilde.town/tildetown/town/stats" ) -const defaultIndexPath = "/etc/skel/public_html/index.html" -const description = `an intentional digital community for creating and sharing -works of art, peer education, and technological anachronism. we are -non-commercial, donation supported, and committed to rejecting false -technological progress in favor of empathy and sustainable computing.` - -type newsEntry struct { - Title string `json:"title"` // Title of entry - Pubdate string `json:"pubdate"` // Human readable date - Content string `json:"content"` // HTML of entry -} - -type user struct { - Username string `json:"username"` // Username of user - PageTitle string `json:"title"` // Title of user's HTML page, if they have one - Mtime int64 `json:"mtime"` // Timestamp representing the last time a user's index.html was modified - // Town additions - DefaultPage bool `json:"default"` // Whether or not user has updated their default index.html -} - -type tildeData struct { - Name string `json:"name"` // Name of the server - URL string `json:"url"` // URL of the server's homepage - SignupURL string `json:"signup_url"` // URL for server's signup page - WantUsers bool `json:"want_users"` // Whether or not new users are being accepted - AdminEmail string `json:"admin_email"` // Email for server admin - Description string `json:"description"` // Description of server - UserCount int `json:"user_count"` // Total number of users on server sorted by last activity time - Users []*user `json:"users"` - // Town Additions - LiveUserCount int `json:"live_user_count"` // Users who have changed their index.html - ActiveUserCount int `json:"active_user_count"` // Users with an active session - ActiveUsers []string `json:"active_users"` // Usernames of logged in users - GeneratedAt string `json:"generated_at"` // When this was generated in '%Y-%m-%d %H:%M:%S' format - GeneratedAtSec int64 `json:"generated_at_sec"` // When this was generated in seconds since epoch - Uptime string `json:"uptime"` // output of `uptime -p` - News []newsEntry `json:"news"` // Collection of town news entries -} - -func getEnvDefault(key, def string) string { - result := os.Getenv(key) - if result == "" { - result = def - } - return result -} - -func homesDir() string { return getEnvDefault("HOMES_DIR", "/home") } - -func getNews() (entries []newsEntry, err error) { - inMeta := true - inContent := false - current := newsEntry{} - blankLineRe := regexp.MustCompile(`^ *\n$`) - - newsPath := getEnvDefault("NEWS_PATH", "/town/news.posts") - - newsFile, err := os.Open(newsPath) - if err != nil { - return entries, fmt.Errorf("unable to read news file: %s", err) - } - defer newsFile.Close() - - scanner := bufio.NewScanner(newsFile) - - for scanner.Scan() { - newsLine := scanner.Text() - if strings.HasPrefix(newsLine, "#") || newsLine == "" || blankLineRe.FindStringIndex(newsLine) != nil { - continue - } else if strings.HasPrefix(newsLine, "--") { - entries = append(entries, current) - current = newsEntry{} - inMeta = true - inContent = false - } else if inMeta { - kv := strings.SplitN(newsLine, ":", 2) - if kv[0] == "pubdate" { - current.Pubdate = strings.TrimSpace(kv[1]) - } else if kv[0] == "title" { - current.Title = strings.TrimSpace(kv[1]) - } else { - log.Printf("ignoring unknown metadata in news entry: %v\n", newsLine) - } - if current.Pubdate != "" && current.Title != "" { - inMeta = false - inContent = true - } - } else if inContent { - current.Content += fmt.Sprintf("\n%v", strings.TrimSpace(newsLine)) - } else { - panic("news post parsing should never reach this point") - } - } - return entries, nil -} - -func indexPathFor(username string) (string, error) { - potentialPaths := []string{"index.html", "index.htm"} - indexPath := "" - errs := []error{} - for _, p := range potentialPaths { - fullPath := path.Join(homesDir(), username, "public_html", p) - _, staterr := os.Stat(fullPath) - if staterr != nil { - errs = append(errs, staterr) - } else { - indexPath = fullPath - break - } - } - - if indexPath == "" { - return "", fmt.Errorf("failed to locate index file for %v; tried %v; encountered errors: %v", username, potentialPaths, errs) - } - - return indexPath, nil -} - -func pageTitleFor(username string) string { - pageTitleRe := regexp.MustCompile(`]*>(.*)`) - indexPath, err := indexPathFor(username) - if err != nil { - return "" - } - content, err := ioutil.ReadFile(indexPath) - if err != nil { - return "" - } - titleMatch := pageTitleRe.FindStringSubmatch(string(content)) - if len(titleMatch) < 2 { - return "" - } - return titleMatch[1] -} - -func systemUsers() map[string]bool { - systemUsers := map[string]bool{ - "ubuntu": true, - "ttadmin": true, - "root": true, - } - envSystemUsers := os.Getenv("SYSTEM_USERS") - if envSystemUsers != "" { - for _, username := range strings.Split(envSystemUsers, ",") { - systemUsers[username] = true - } - } - - return systemUsers -} - -func mtimeFor(username string) int64 { - path := path.Join(homesDir(), username, "public_html") - var maxMtime int64 = 0 - _ = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if maxMtime < info.ModTime().Unix() { - maxMtime = info.ModTime().Unix() - } - return nil - }) - - return maxMtime -} - -func detectDefaultPageFor(username string, defaultHTML []byte) bool { - indexPath, err := indexPathFor(username) - if err != nil { - return false - } - indexFile, err := os.Open(indexPath) - if err != nil { - return false - } - defer indexFile.Close() - - indexHTML, err := ioutil.ReadAll(indexFile) - if err != nil { - return false - } - return bytes.Equal(indexHTML, defaultHTML) -} - -func getDefaultHTML() ([]byte, error) { - indexPath := os.Getenv("DEFAULT_INDEX_PATH") - if indexPath == "" { - indexPath = defaultIndexPath - } - - defaultIndexFile, err := os.Open(indexPath) - if err != nil { - return []byte{}, fmt.Errorf("could not open default index: %s", err) - } - defer defaultIndexFile.Close() - - defaultIndexHTML, err := ioutil.ReadAll(defaultIndexFile) - if err != nil { - return []byte{}, fmt.Errorf("could not read default index: %s", err) - } - - return defaultIndexHTML, nil -} - -type byMtime []*user - -func (x byMtime) Len() int { return len(x) } -func (x byMtime) Less(i, j int) bool { return x[i].Mtime > x[j].Mtime } // because we want DESC -func (x byMtime) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func getUsers() (users []*user, err error) { - // For the purposes of this program, we discover users via: - // - presence in /home/ - // - absence in systemUsers list (sourced from source code and potentially augmented by an environment variable) - // We formally used passwd parsing. This is definitely more "correct" and I'm - // not opposed to going back to that; going back to parsing /home is mainly to - // get this new version going. - defaultIndexHTML, err := getDefaultHTML() - if err != nil { - return users, err - } - - out, err := exec.Command("ls", homesDir()).Output() - if err != nil { - return users, fmt.Errorf("could not run ls: %s", err) - } - - scanner := bufio.NewScanner(bytes.NewReader(out)) - - systemUsers := systemUsers() - - for scanner.Scan() { - username := scanner.Text() - if systemUsers[username] { - continue - } - user := user{ - Username: username, - PageTitle: pageTitleFor(username), - Mtime: mtimeFor(username), - DefaultPage: detectDefaultPageFor(username, defaultIndexHTML), - } - users = append(users, &user) - } - - return users, nil -} - -func liveUserCount(users []*user) int { - count := 0 - for _, u := range users { - if !u.DefaultPage { - count++ - } - } - return count -} - -func activeUsers() ([]string, error) { - out, err := exec.Command("sh", "-c", "who | cut -d' ' -f1 | sort -u").Output() - if err != nil { - return nil, fmt.Errorf("failed to get active user count: %w", err) - } - scanner := bufio.NewScanner(bytes.NewReader(out)) - usernames := []string{} - for scanner.Scan() { - usernames = append(usernames, strings.TrimSpace(scanner.Text())) - } - return usernames, nil -} - -func getUptime() (string, error) { - out, err := exec.Command("uptime").Output() - if err != nil { - return "", fmt.Errorf("could not run uptime: %s", err) - } - return strings.TrimSpace(string(out)), nil -} - -func tdp() (tildeData, error) { - users, err := getUsers() - if err != nil { - return tildeData{}, fmt.Errorf("could not get user list: %s", err) - } - activeUsernames, err := activeUsers() - if err != nil { - return tildeData{}, fmt.Errorf("could not count active users: %s", err) - } - news, err := getNews() - if err != nil { - return tildeData{}, fmt.Errorf("could not get news: %s", err) - } - - uptime, err := getUptime() - if err != nil { - return tildeData{}, fmt.Errorf("could not determine uptime: %s", err) - } - - sort.Sort(byMtime(users)) - - return tildeData{ - Name: "tilde.town", - URL: "https://tilde.town", - SignupURL: "https://cgi.tilde.town/users/signup", - WantUsers: true, - AdminEmail: "root@tilde.town", - Description: description, - UserCount: len(users), - Users: users, - LiveUserCount: liveUserCount(users), - ActiveUserCount: len(activeUsernames), - ActiveUsers: activeUsernames, - Uptime: uptime, - News: news, - GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"), - GeneratedAtSec: time.Now().Unix(), - }, nil -} - func main() { - systemData, err := tdp() + systemData, err := stats.Stats() if err != nil { log.Fatal(err) } diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..24290d5 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,338 @@ +package stats + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +const defaultIndexPath = "/etc/skel/public_html/index.html" +const description = `an intentional digital community for creating and sharing +works of art, peer education, and technological anachronism. we are +non-commercial, donation supported, and committed to rejecting false +technological progress in favor of empathy and sustainable computing.` + +type NewsEntry struct { + Title string `json:"title"` // Title of entry + Pubdate string `json:"pubdate"` // Human readable date + Content string `json:"content"` // HTML of entry +} + +type User struct { + Username string `json:"username"` // Username of user + PageTitle string `json:"title"` // Title of user's HTML page, if they have one + Mtime int64 `json:"mtime"` // Timestamp representing the last time a user's index.html was modified + // Town additions + DefaultPage bool `json:"default"` // Whether or not user has updated their default index.html +} + +type TildeData struct { + Name string `json:"name"` // Name of the server + URL string `json:"url"` // URL of the server's homepage + SignupURL string `json:"signup_url"` // URL for server's signup page + WantUsers bool `json:"want_users"` // Whether or not new users are being accepted + AdminEmail string `json:"admin_email"` // Email for server admin + Description string `json:"description"` // Description of server + UserCount int `json:"user_count"` // Total number of users on server sorted by last activity time + Users []*User `json:"users"` + + // Town Additions + LiveUserCount int `json:"live_user_count"` // Users who have changed their index.html + ActiveUserCount int `json:"active_user_count"` // Users with an active session + ActiveUsers []string `json:"active_users"` // Usernames of logged in users + GeneratedAt string `json:"generated_at"` // When this was generated in '%Y-%m-%d %H:%M:%S' format + GeneratedAtSec int64 `json:"generated_at_sec"` // When this was generated in seconds since epoch + Uptime string `json:"uptime"` // output of `uptime -p` + News []NewsEntry `json:"news"` // Collection of town news entries +} + +func getEnvDefault(key, def string) string { + result := os.Getenv(key) + if result == "" { + result = def + } + return result +} + +func homesDir() string { return getEnvDefault("HOMES_DIR", "/home") } + +func getNews() (entries []NewsEntry, err error) { + inMeta := true + inContent := false + current := NewsEntry{} + blankLineRe := regexp.MustCompile(`^ *\n$`) + + newsPath := getEnvDefault("NEWS_PATH", "/town/news.posts") + + newsFile, err := os.Open(newsPath) + if err != nil { + return entries, fmt.Errorf("unable to read news file: %s", err) + } + defer newsFile.Close() + + scanner := bufio.NewScanner(newsFile) + + for scanner.Scan() { + newsLine := scanner.Text() + if strings.HasPrefix(newsLine, "#") || newsLine == "" || blankLineRe.FindStringIndex(newsLine) != nil { + continue + } else if strings.HasPrefix(newsLine, "--") { + entries = append(entries, current) + current = NewsEntry{} + inMeta = true + inContent = false + } else if inMeta { + kv := strings.SplitN(newsLine, ":", 2) + if kv[0] == "pubdate" { + current.Pubdate = strings.TrimSpace(kv[1]) + } else if kv[0] == "title" { + current.Title = strings.TrimSpace(kv[1]) + } else { + log.Printf("ignoring unknown metadata in news entry: %v\n", newsLine) + } + if current.Pubdate != "" && current.Title != "" { + inMeta = false + inContent = true + } + } else if inContent { + current.Content += fmt.Sprintf("\n%v", strings.TrimSpace(newsLine)) + } else { + panic("news post parsing should never reach this point") + } + } + return entries, nil +} + +func indexPathFor(username string) (string, error) { + potentialPaths := []string{"index.html", "index.htm"} + indexPath := "" + errs := []error{} + for _, p := range potentialPaths { + fullPath := path.Join(homesDir(), username, "public_html", p) + _, staterr := os.Stat(fullPath) + if staterr != nil { + errs = append(errs, staterr) + } else { + indexPath = fullPath + break + } + } + + if indexPath == "" { + return "", fmt.Errorf("failed to locate index file for %v; tried %v; encountered errors: %v", username, potentialPaths, errs) + } + + return indexPath, nil +} + +func pageTitleFor(username string) string { + pageTitleRe := regexp.MustCompile(`]*>(.*)`) + indexPath, err := indexPathFor(username) + if err != nil { + return "" + } + content, err := ioutil.ReadFile(indexPath) + if err != nil { + return "" + } + titleMatch := pageTitleRe.FindStringSubmatch(string(content)) + if len(titleMatch) < 2 { + return "" + } + return titleMatch[1] +} + +func systemUsers() map[string]bool { + systemUsers := map[string]bool{ + "ubuntu": true, + "ttadmin": true, + "root": true, + } + envSystemUsers := os.Getenv("SYSTEM_USERS") + if envSystemUsers != "" { + for _, username := range strings.Split(envSystemUsers, ",") { + systemUsers[username] = true + } + } + + return systemUsers +} + +func mtimeFor(username string) int64 { + path := path.Join(homesDir(), username, "public_html") + var maxMtime int64 = 0 + _ = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if maxMtime < info.ModTime().Unix() { + maxMtime = info.ModTime().Unix() + } + return nil + }) + + return maxMtime +} + +func detectDefaultPageFor(username string, defaultHTML []byte) bool { + indexPath, err := indexPathFor(username) + if err != nil { + return false + } + indexFile, err := os.Open(indexPath) + if err != nil { + return false + } + defer indexFile.Close() + + indexHTML, err := ioutil.ReadAll(indexFile) + if err != nil { + return false + } + return bytes.Equal(indexHTML, defaultHTML) +} + +func getDefaultHTML() ([]byte, error) { + indexPath := os.Getenv("DEFAULT_INDEX_PATH") + if indexPath == "" { + indexPath = defaultIndexPath + } + + defaultIndexFile, err := os.Open(indexPath) + if err != nil { + return []byte{}, fmt.Errorf("could not open default index: %s", err) + } + defer defaultIndexFile.Close() + + defaultIndexHTML, err := ioutil.ReadAll(defaultIndexFile) + if err != nil { + return []byte{}, fmt.Errorf("could not read default index: %s", err) + } + + return defaultIndexHTML, nil +} + +type byMtime []*User + +func (x byMtime) Len() int { return len(x) } +func (x byMtime) Less(i, j int) bool { return x[i].Mtime > x[j].Mtime } // because we want DESC +func (x byMtime) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func getUsers() (users []*User, err error) { + // For the purposes of this program, we discover users via: + // - presence in /home/ + // - absence in systemUsers list (sourced from source code and potentially augmented by an environment variable) + // We formally used passwd parsing. This is definitely more "correct" and I'm + // not opposed to going back to that; going back to parsing /home is mainly to + // get this new version going. + defaultIndexHTML, err := getDefaultHTML() + if err != nil { + return users, err + } + + out, err := exec.Command("ls", homesDir()).Output() + if err != nil { + return users, fmt.Errorf("could not run ls: %s", err) + } + + scanner := bufio.NewScanner(bytes.NewReader(out)) + + systemUsers := systemUsers() + + for scanner.Scan() { + username := scanner.Text() + if systemUsers[username] { + continue + } + user := User{ + Username: username, + PageTitle: pageTitleFor(username), + Mtime: mtimeFor(username), + DefaultPage: detectDefaultPageFor(username, defaultIndexHTML), + } + users = append(users, &user) + } + + return users, nil +} + +func liveUserCount(users []*User) int { + count := 0 + for _, u := range users { + if !u.DefaultPage { + count++ + } + } + return count +} + +func activeUsers() ([]string, error) { + out, err := exec.Command("sh", "-c", "who | cut -d' ' -f1 | sort -u").Output() + if err != nil { + return nil, fmt.Errorf("failed to get active user count: %w", err) + } + scanner := bufio.NewScanner(bytes.NewReader(out)) + usernames := []string{} + for scanner.Scan() { + usernames = append(usernames, strings.TrimSpace(scanner.Text())) + } + return usernames, nil +} + +func getUptime() (string, error) { + out, err := exec.Command("uptime").Output() + if err != nil { + return "", fmt.Errorf("could not run uptime: %s", err) + } + return strings.TrimSpace(string(out)), nil +} + +func Stats() (TildeData, error) { + users, err := getUsers() + if err != nil { + return TildeData{}, fmt.Errorf("could not get user list: %s", err) + } + activeUsernames, err := activeUsers() + if err != nil { + return TildeData{}, fmt.Errorf("could not count active users: %s", err) + } + news, err := getNews() + if err != nil { + return TildeData{}, fmt.Errorf("could not get news: %s", err) + } + + uptime, err := getUptime() + if err != nil { + return TildeData{}, fmt.Errorf("could not determine uptime: %s", err) + } + + sort.Sort(byMtime(users)) + + return TildeData{ + Name: "tilde.town", + URL: "https://tilde.town", + SignupURL: "https://cgi.tilde.town/users/signup", + WantUsers: true, + AdminEmail: "root@tilde.town", + Description: description, + UserCount: len(users), + Users: users, + LiveUserCount: liveUserCount(users), + ActiveUserCount: len(activeUsernames), + ActiveUsers: activeUsernames, + Uptime: uptime, + News: news, + GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"), + GeneratedAtSec: time.Now().Unix(), + }, nil +}