let's fetch some feeds

master
magical 2021-11-15 03:32:24 +00:00
parent cb8d99214a
commit ce9960c06a
2 changed files with 100 additions and 0 deletions

2
go.mod
View File

@ -1,3 +1,5 @@
module git.tilde.town/magical/feedget module git.tilde.town/magical/feedget
go 1.14 go 1.14
require github.com/mmcdole/gofeed v1.1.3

98
main.go
View File

@ -1,3 +1,101 @@
// Feedget scrapes RSS feeds (and other sources) // Feedget scrapes RSS feeds (and other sources)
// and spits the latest headline from each onto a static web page. // and spits the latest headline from each onto a static web page.
package main package main
import (
"context"
"fmt"
"log"
"sort"
"sync"
"time"
"github.com/mmcdole/gofeed"
)
var feedurls = []string{
"https://tilde.team/~dozens/dreams/rss.xml",
//"https://xkcd.com/atom.xml",
"https://tilde.town/~magical/xkcd.xml",
}
func main() {
var sources = make([]*FeedSource, len(feedurls)) // TODO: interface Source
for i, u := range feedurls {
sources[i] = NewFeed(u)
}
var wg sync.WaitGroup
wg.Add(len(sources))
for i := range sources {
src := sources[i]
go func() {
src.update()
wg.Done()
}()
}
wg.Wait()
for _, src := range sources {
fmt.Println(src.Title, src.Error)
}
//var feeds []*gofeed.Feed
//var errors []error
//for _, url := range feedUrls {
//}
}
type Source interface {
Title() string
Link() string
Error() error
Update(context.Context)
}
// want to keep track of:
// - whether the most recent update suceeded
// - when the last successful update was
// - how many of the last N updates succeeded
// - status codes for the last N updates
// - response time for the last N updates
// - how frequently items are posted
type Cache struct {
}
type FeedSource struct {
Items []*gofeed.Item
Title string
URL string
LastFetch time.Time
Error error
mu sync.Mutex
}
func NewFeed(url string) *FeedSource {
return &FeedSource{
URL: url,
}
}
func (src *FeedSource) update() {
src.mu.Lock()
defer src.mu.Unlock()
fp := gofeed.NewParser()
feed, err := fp.ParseURL(src.URL)
if err != nil {
err := fmt.Errorf("error parsing %q: %v", src.URL, err)
log.Println(err)
src.Error = err
return // return err?
}
items := feed.Items
sort.Slice(items, func(i, j int) bool {
return items[i].Updated >= items[j].Updated
})
src.Title = feed.Title
src.Items = items
src.LastFetch = time.Now()
}