diff --git a/main.go b/main.go index 02ad8cc..1bc6741 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "log" + "net/http" "sort" "sync" "time" @@ -13,6 +14,8 @@ import ( "github.com/mmcdole/gofeed" ) +const UserAgent = "feedget/0.1" + func main() { var sources = []*FeedSource{ // TODO: interface Source NewFeed("https://tilde.team/~dozens/dreams/rss.xml"), @@ -31,7 +34,7 @@ func main() { wg.Wait() for _, src := range sources { - fmt.Println(src.Title, src.Error) + fmt.Println(src.Title, src.Error, src.LastStatus) } } @@ -60,6 +63,9 @@ type FeedSource struct { LastFetch time.Time Error error + LastStatusCode int + LastStatus string + mu sync.Mutex } @@ -73,13 +79,54 @@ func (src *FeedSource) update() { src.mu.Lock() defer src.mu.Unlock() fp := gofeed.NewParser() - feed, err := fp.ParseURL(src.URL) + + ctx := context.TODO() + + req, err := http.NewRequest("GET", src.URL, nil) if err != nil { - err := fmt.Errorf("error parsing %q: %v", src.URL, err) + src.Error = fmt.Errorf("error fetching %q: %w", src.URL, err) + log.Println(src.Error) + return // return err? + } + req = req.WithContext(ctx) + req.Header.Set("User-Agent", UserAgent) + // TODO: If-Modified-Since, Etag + + resp, err := http.DefaultClient.Do(req) + if err != nil { + err := fmt.Errorf("error fetching %q: %w", src.URL, err) log.Println(err) src.Error = err return // return err? } + + if resp != nil && resp.Body != nil { + defer func() { + err := resp.Body.Close() + if err != nil { + log.Printf("error closing response body for %q: %v", src.URL, err) + } + }() + } + + src.LastStatusCode = resp.StatusCode + src.LastStatus = resp.Status + if resp.StatusCode != 200 { + src.Error = fmt.Errorf("error fetching %q: status %s", src.URL, resp.Status) + log.Println(src.Error) + return + } + + // TODO: cache body + + feed, err := fp.Parse(resp.Body) + if err != nil { + err := fmt.Errorf("error parsing %q: %w", src.URL, err) + log.Println(err) + src.Error = err + return // return err? + } + items := feed.Items sort.Slice(items, func(i, j int) bool { return items[i].Updated >= items[j].Updated @@ -87,4 +134,5 @@ func (src *FeedSource) update() { src.Title = feed.Title src.Items = items src.LastFetch = time.Now() + src.Error = nil }