From 7add59d14ab9f622cc20d4a23cb82bb6e01134c8 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 5 May 2024 14:08:06 +0200 Subject: [PATCH] Add sync service --- api/cmd/main.go | 7 +++- api/entries.go | 16 ++++++-- api/feeds.go | 61 +++++++++++++++------------- api/{rss.go => reader.go} | 0 api/sync.go | 83 +++++++++++++++++++++++++++++++++++++++ sql/create.sql | 4 +- 6 files changed, 137 insertions(+), 34 deletions(-) rename api/{rss.go => reader.go} (100%) create mode 100644 api/sync.go diff --git a/api/cmd/main.go b/api/cmd/main.go index 49e544d..2421627 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -47,12 +47,17 @@ func main() { if err != nil { log.Fatal(err) } + + reader := vex.NewRssReader(http.DefaultClient) h := Handler{ - feeds: vex.NewFeedService(db), + feeds: vex.NewFeedService(db, &reader), entries: vex.NewEntryService(db), users: vex.NewUserService(db), jwtSecret: []byte(os.Getenv("JWT_SECRET")), } + sync := vex.NewSyncService(&reader, &h.feeds, &h.entries) + + go sync.SyncFeedsForever() e := echo.New() e.Validator = &Validator{validator: validator.New()} diff --git a/api/entries.go b/api/entries.go index 6aa8321..de53965 100644 --- a/api/entries.go +++ b/api/entries.go @@ -5,6 +5,7 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" + "github.com/lib/pq" ) type Entry struct { @@ -13,7 +14,7 @@ type Entry struct { Link string `json:"link"` Date time.Time `json:"date"` Content string `json:"content"` - Author *string `json:"author"` + Authors []string `json:"authors"` FeedId uuid.UUID `json:"feedId"` Feed Feed `json:"feed,omitempty"` } @@ -24,7 +25,7 @@ type EntryDao struct { Link string Date time.Time Content string - Author *string + Authors pq.StringArray FeedId uuid.UUID `db:"feed_id"` Feed FeedDao `db:"feed"` } @@ -36,7 +37,7 @@ func (e *EntryDao) ToEntry() Entry { Link: e.Link, Date: e.Date, Content: e.Content, - Author: e.Author, + Authors: e.Authors, FeedId: e.FeedId, Feed: e.Feed.ToFeed(), } @@ -50,6 +51,15 @@ func NewEntryService(db *sqlx.DB) EntryService { return EntryService{database: db} } +func (s EntryService) Add(entries []EntryDao) error { + _, err := s.database.NamedExec( + `insert into entries (id, title, link, date, content, authors, feed_id) + values (:id, :title, :link, :date, :content, :authors, :feed_id)`, + entries, + ) + return err +} + func (s EntryService) ListEntries() ([]Entry, error) { ret := []EntryDao{} err := s.database.Select( diff --git a/api/feeds.go b/api/feeds.go index 3724a62..8506264 100644 --- a/api/feeds.go +++ b/api/feeds.go @@ -2,7 +2,6 @@ package vex import ( "fmt" - "net/http" "time" "github.com/google/uuid" @@ -11,14 +10,16 @@ import ( ) type Feed struct { - Id uuid.UUID `json:"id"` - Name string `json:"name"` - Link string `json:"link"` - FaviconUrl string `json:"faviconUrl"` - Tags []string `json:"tags"` - SubmitterId uuid.UUID `json:"submitterId"` - Submitter *User `json:"submitter,omitempty"` - AddedDate time.Time `json:"addedDate"` + Id uuid.UUID `json:"id"` + Name string `json:"name"` + Link string `json:"link"` + FaviconUrl string `json:"faviconUrl"` + Tags []string `json:"tags"` + SubmitterId uuid.UUID `json:"submitterId"` + Submitter *User `json:"submitter,omitempty"` + AddedDate time.Time `json:"addedDate"` + etag string + lastFetchDate *time.Time } type FeedDao struct { @@ -36,39 +37,43 @@ type FeedDao struct { func (f *FeedDao) ToFeed() Feed { return Feed{ - Id: f.Id, - Name: f.Name, - Link: f.Link, - FaviconUrl: f.FaviconUrl, - Tags: f.Tags, - SubmitterId: f.SubmitterId, - Submitter: f.Submitter, - AddedDate: f.AddedDate, + Id: f.Id, + Name: f.Name, + Link: f.Link, + FaviconUrl: f.FaviconUrl, + Tags: f.Tags, + SubmitterId: f.SubmitterId, + Submitter: f.Submitter, + AddedDate: f.AddedDate, + etag: f.Etag, + lastFetchDate: f.LastFetchDate, } } func (f *Feed) ToDao() FeedDao { return FeedDao{ - Id: f.Id, - Name: f.Name, - Link: f.Link, - FaviconUrl: f.FaviconUrl, - Tags: f.Tags, - SubmitterId: f.SubmitterId, - Submitter: f.Submitter, - AddedDate: f.AddedDate, + Id: f.Id, + Name: f.Name, + Link: f.Link, + FaviconUrl: f.FaviconUrl, + Tags: f.Tags, + SubmitterId: f.SubmitterId, + Submitter: f.Submitter, + AddedDate: f.AddedDate, + Etag: f.etag, + LastFetchDate: f.lastFetchDate, } } type FeedService struct { database *sqlx.DB - reader Reader + reader *Reader } -func NewFeedService(db *sqlx.DB) FeedService { +func NewFeedService(db *sqlx.DB, reader *Reader) FeedService { return FeedService{ database: db, - reader: NewRssReader(http.DefaultClient), + reader: reader, } } diff --git a/api/rss.go b/api/reader.go similarity index 100% rename from api/rss.go rename to api/reader.go diff --git a/api/sync.go b/api/sync.go new file mode 100644 index 0000000..376334b --- /dev/null +++ b/api/sync.go @@ -0,0 +1,83 @@ +package vex + +import ( + "cmp" + "log" + "time" + + "github.com/google/uuid" + "github.com/mmcdole/gofeed" +) + +type SyncService struct { + reader *Reader + feeds *FeedService + entries *EntryService +} + +func NewSyncService(reader *Reader, feeds *FeedService, entries *EntryService) SyncService { + return SyncService{ + reader: reader, + feeds: feeds, + entries: entries, + } +} + +func (s SyncService) SyncFeed(feed Feed) error { + info, err := s.reader.ReadFeed(feed.Link, feed.etag, feed.lastFetchDate) + if err != nil { + return err + } + if info == nil { + // no new items + return nil + } + log.Printf("Adding %v new entries", len(info.Items)) + entries := Map(info.Items, func(item *gofeed.Item, _ int) EntryDao { + var date time.Time + if item.PublishedParsed != nil { + date = *item.PublishedParsed + } else { + date = time.Now() + } + + return EntryDao{ + Id: uuid.New(), + Title: item.Title, + Link: item.Link, + Date: date, + Authors: Map(item.Authors, func(author *gofeed.Person, _ int) string { return author.Name }), + Content: cmp.Or(item.Content, item.Description), + FeedId: feed.Id, + } + }) + err = s.entries.Add(entries) + if err != nil { + return err + } + // TODO: update etag and last fetch date of feed + return nil +} + +func (s SyncService) SyncFeeds() error { + feeds, err := s.feeds.ListFeeds() + if err != nil { + log.Printf("Could not retrive feeds: %v", err) + return err + } + for _, feed := range feeds { + err := s.SyncFeed(feed) + if err != nil { + log.Printf("Could not sync feed %v: %v", feed.Link, err) + // TODO: s.feeds.SaveError(feed.Id, err) + } + } + return nil +} + +func (s SyncService) SyncFeedsForever() { + for { + s.SyncFeeds() + time.Sleep(15 * time.Minute) + } +} diff --git a/sql/create.sql b/sql/create.sql index 7912b1d..38af26a 100644 --- a/sql/create.sql +++ b/sql/create.sql @@ -24,7 +24,7 @@ create table if not exists entries( link text not null, date timestamp with time zone not null, content text not null, - author text + authors text[] not null ); create table if not exists entries_users( @@ -34,6 +34,6 @@ create table if not exists entries_users( is_bookmarked bool not null, is_read_later bool not null, is_ignored bool not null, - constraint entries_users_pk primary key(user_id, feed_id) + constraint entries_users_pk primary key (user_id, feed_id) );