package main import ( "bytes" "context" "fmt" "io" "log" "net/http" "sort" "strings" "sync" "time" "github.com/PuerkitoBio/goquery" "golang.org/x/net/html" "golang.org/x/net/html/atom" ) const maxResponseSize = 1e6 // 1MB // https://docs.joinmastodon.org/spec/microformats/ // Root elements (h-*) // // h-feed // Represents a stream of entries. Attached to a profile's toots. Also // attached to the parent thread within detailed status views. // // h-entry // Represents episodic or date stamped online content. Attached to a status. // // URL properties (u-*) // // u-photo // Within h-card, represents the profile picture. Attached to the avatar image. // // u-uid // Within h-entry or h-cite, represents a universally unique identifier. // Attached to timestamp link. // // u-url // Within h-entry or h-cite, represents the status permalink. Attached to // timestamp link. Within h-card, represents the profile permalink. // Attached to display name link. // // Datetime properties (dt-*) // // dt-published // Within h-entry or h-cite, represents the date and time at which the // status was published. Attached to data element with value attribute. // // Element tree (e-*) // // e-content // Within h-entry or h-cite, represents the content of the status. Attached to status content. // type MastoSource struct { Items []*MastoItem Title string URL string LastFetch time.Time Error error LastStatusCode int LastStatus string mu sync.Mutex } var _ Source = &MastoSource{} type MastoFeed struct { Title string Items []*MastoItem } type MastoItem struct { Title string `json:"title,omitempty"` Content string `json:"content,omitempty"` Link string `json:"link,omitempty"` PublishedString string `json:"published,omitempty"` Author string `json:"author,omitempty"` IsBoost bool `json:"is_boost,omitempty"` } func NewMastoSource(url string) *MastoSource { return &MastoSource{ URL: url, } } func (src *MastoSource) update(ctx context.Context) { src.mu.Lock() defer src.mu.Unlock() req, err := http.NewRequestWithContext(ctx, "GET", src.URL, nil) if err != nil { src.Error = fmt.Errorf("error fetching %q: %w", src.URL, err) log.Println(src.Error) return // return err? } req.Header.Set("User-Agent", UserAgent) // TODO: If-Modified-Since, Etag resp, err := http.DefaultClient.Do(req) if err != nil { err := fmt.Errorf("error fetching %q: %w", src.URL, err) log.Println(err) src.Error = err return // return err? } if resp != nil && resp.Body != nil { defer func() { err := resp.Body.Close() if err != nil { log.Printf("error closing response body for %q: %v", src.URL, err) } }() } src.LastStatusCode = resp.StatusCode src.LastStatus = resp.Status if resp.StatusCode != 200 { src.Error = fmt.Errorf("error fetching %q: status %s", src.URL, resp.Status) log.Println(src.Error) return } // TODO: cache body body := MaxBytesReader(resp.Body, maxResponseSize) feed, err := parseMicroformats(body) if err != nil { err := fmt.Errorf("error parsing %q: %w", src.URL, err) log.Println(err) src.Error = err return // return err? } items := feed.Items sort.Slice(items, func(i, j int) bool { return items[i].PublishedString >= items[j].PublishedString }) src.Title = feed.Title src.Items = items src.LastFetch = time.Now() src.Error = nil } func parseMicroformats(r io.Reader) (*MastoFeed, error) { doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, err } feed := new(MastoFeed) if doc.Find(".h-feed").Length() == 0 { return nil, fmt.Errorf("no feed content found") } feed.Title = doc.Find(".h-feed > .p-name").First().AttrOr("value", "") doc.Find(".h-feed").Find(".h-entry, .h-cite").Each(func(i int, elem *goquery.Selection) { cw := strings.TrimSpace(text(elem.Find(".p-summary").First())) // TODO: move this logic to GetItems if cw != "" { cw = "[" + cw + "] " } feed.Items = append(feed.Items, &MastoItem{ Title: "", Content: cw + text(elem.Find(".e-content").First()), Link: elem.Find("a.u-url.u-uid").AttrOr("href", ""), Author: text(elem.Find(".p-author .p-name").First()), PublishedString: elem.Find("data.dt-published").AttrOr("value", ""), IsBoost: elem.HasClass("h-cite"), }) }) return feed, nil } // Text gets the combined text contents of each element in the set of matched // elements, including their descendants. func text(s *goquery.Selection) string { var buf bytes.Buffer // Slightly optimized vs calling Each: no single selection object created var f func(*html.Node) f = func(n *html.Node) { if n.Type == html.TextNode { // Keep newlines and spaces, like jQuery buf.WriteString(n.Data) } else if n.Type == html.ElementNode && n.DataAtom == atom.Br { //buf.WriteString("\n") buf.WriteString(" ") } else if n.Type == html.ElementNode && n.DataAtom == atom.P && n.PrevSibling != nil { //buf.WriteString("\n\n") buf.WriteString(" ") } else if n.Type == html.ElementNode && (n.DataAtom == atom.Script || n.DataAtom == atom.Style || n.DataAtom == atom.Template) { // nothing } if n.FirstChild != nil { for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } } for _, n := range s.Nodes { f(n) } return buf.String() } func (src *MastoSource) GetTitle() string { return src.Title } func (src *MastoSource) GetError() error { return src.Error } func (src *MastoSource) GetItems() (items []Item) { for _, x := range src.Items { text := x.Content if x.IsBoost { text = "RT @" + x.Author + ": " + text } d, _ := time.Parse(time.RFC3339, x.PublishedString) items = append(items, Item{ Date: d, Link: x.Link, Text: text, }) } return }