Merge pull request #9 from zoriya/api

Add rss parsing/sync
2026-06-03 18:51:51 +00:00 · 2024-05-05 15:26:12 +02:00
parent 3b782e3991 9c688fef18
commit 9f753d118e
12 changed files with 354 additions and 49 deletions
@@ -2,5 +2,8 @@ FROM golang:1.22-alpine
 RUN go install github.com/bokwoon95/wgo@latest
 WORKDIR /app

+COPY go.mod go.sum ./
+RUN go mod download
+
 EXPOSE 1597
 CMD wgo run ./cmd
@@ -1,6 +1,7 @@
 package main

 import (
+	"fmt"
 	"log"
 	"net/http"

@@ -35,7 +36,17 @@ func (h *Handler) AddFeed(c echo.Context) error {
 		return err
 	}

-	feed, err := h.feeds.AddFeed(req.Link, req.Tags, user)
+	feeds, err := h.feeds.GetFeedData(req.Link)
+	if err != nil {
+		return echo.NewHTTPError(400, fmt.Sprintf("Invalid feed link: %v", err))
+	}
+	if len(feeds) != 1 {
+		return c.JSON(409, feeds)
+	}
+	feed := feeds[0]
+	feed.SubmitterId = user
+	feed.Tags = req.Tags
+	feed, err = h.feeds.AddFeed(feed)
 	if err != nil {
 		log.Printf("Add feed error: %v", err)
 		return echo.NewHTTPError(500, "internal server error")
@@ -47,12 +47,17 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
+	reader := vex.NewRssReader(http.DefaultClient)
 	h := Handler{
-		feeds:     vex.NewFeedService(db),
+		feeds:     vex.NewFeedService(db, &reader),
 		entries:   vex.NewEntryService(db),
 		users:     vex.NewUserService(db),
 		jwtSecret: []byte(os.Getenv("JWT_SECRET")),
 	}
+	sync := vex.NewSyncService(&reader, &h.feeds, &h.entries)
+
+	go sync.SyncFeedsForever()

 	e := echo.New()
 	e.Validator = &Validator{validator: validator.New()}
@@ -5,6 +5,7 @@ import (

 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
 )

 type Entry struct {
@@ -13,7 +14,7 @@ type Entry struct {
 	Link    string    `json:"link"`
 	Date    time.Time `json:"date"`
 	Content string    `json:"content"`
-	Author  *string   `json:"author"`
+	Authors []string  `json:"authors"`
 	FeedId  uuid.UUID `json:"feedId"`
 	Feed    Feed      `json:"feed,omitempty"`
 }
@@ -24,7 +25,7 @@ type EntryDao struct {
 	Link    string
 	Date    time.Time
 	Content string
-	Author  *string
+	Authors pq.StringArray
 	FeedId  uuid.UUID `db:"feed_id"`
 	Feed    FeedDao   `db:"feed"`
 }
@@ -36,7 +37,7 @@ func (e *EntryDao) ToEntry() Entry {
 		Link:    e.Link,
 		Date:    e.Date,
 		Content: e.Content,
-		Author:  e.Author,
+		Authors: e.Authors,
 		FeedId:  e.FeedId,
 		Feed:    e.Feed.ToFeed(),
 	}
@@ -50,6 +51,15 @@ func NewEntryService(db *sqlx.DB) EntryService {
 	return EntryService{database: db}
 }

+func (s EntryService) Add(entries []EntryDao) error {
+	_, err := s.database.NamedExec(
+		`insert into entries (id, title, link, date, content, authors, feed_id)
+		values (:id, :title, :link, :date, :content, :authors, :feed_id)`,
+		entries,
+	)
+	return err
+}
+
 func (s EntryService) ListEntries() ([]Entry, error) {
 	ret := []EntryDao{}
 	err := s.database.Select(
@@ -1,6 +1,7 @@
 package vex

 import (
+	"fmt"
 	"time"

 	"github.com/google/uuid"
@@ -9,75 +10,110 @@ import (
 )

 type Feed struct {
-	Id          uuid.UUID `json:"id"`
-	Name        string    `json:"name"`
-	Link        string    `json:"link"`
-	FaviconUrl  string    `json:"faviconUrl"`
-	Tags        []string  `json:"tags"`
-	SubmitterId uuid.UUID `json:"submitterId"`
-	Submitter   User      `json:"submitter,omitempty"`
-	AddedDate   time.Time `json:"addedDate"`
+	Id            uuid.UUID `json:"id"`
+	Name          string    `json:"name"`
+	Link          string    `json:"link"`
+	FaviconUrl    string    `json:"faviconUrl"`
+	Tags          []string  `json:"tags"`
+	SubmitterId   uuid.UUID `json:"submitterId"`
+	Submitter     *User     `json:"submitter,omitempty"`
+	AddedDate     time.Time `json:"addedDate"`
+	SyncErorr     *string   `json:"syncError,omitempty"`
+	etag          string
+	lastFetchDate *time.Time
 }

 type FeedDao struct {
-	Id          uuid.UUID
-	Name        string
-	Link        string
-	FaviconUrl  string `db:"favicon_url"`
-	Tags        pq.StringArray
-	SubmitterId uuid.UUID `db:"submitter_id"`
-	Submitter   User      `db:"submitter"`
-	AddedDate   time.Time `db:"added_date"`
+	Id            uuid.UUID
+	Name          string
+	Link          string
+	FaviconUrl    string `db:"favicon_url"`
+	Tags          pq.StringArray
+	SubmitterId   uuid.UUID `db:"submitter_id"`
+	Submitter     *User     `db:"submitter"`
+	AddedDate     time.Time `db:"added_date"`
+	SyncErorr     *string   `db:"sync_error"`
+	Etag          string
+	LastFetchDate *time.Time `db:"last_fetch_date"`
 }

 func (f *FeedDao) ToFeed() Feed {
 	return Feed{
-		Id:          f.Id,
-		Name:        f.Name,
-		Link:        f.Name,
-		FaviconUrl:  f.FaviconUrl,
-		Tags:        f.Tags,
-		SubmitterId: f.SubmitterId,
-		Submitter:   f.Submitter,
-		AddedDate:   f.AddedDate,
+		Id:            f.Id,
+		Name:          f.Name,
+		Link:          f.Link,
+		FaviconUrl:    f.FaviconUrl,
+		Tags:          f.Tags,
+		SubmitterId:   f.SubmitterId,
+		Submitter:     f.Submitter,
+		AddedDate:     f.AddedDate,
+		etag:          f.Etag,
+		lastFetchDate: f.LastFetchDate,
+	}
+}
+
+func (f *Feed) ToDao() FeedDao {
+	return FeedDao{
+		Id:            f.Id,
+		Name:          f.Name,
+		Link:          f.Link,
+		FaviconUrl:    f.FaviconUrl,
+		Tags:          f.Tags,
+		SubmitterId:   f.SubmitterId,
+		Submitter:     f.Submitter,
+		AddedDate:     f.AddedDate,
+		Etag:          f.etag,
+		LastFetchDate: f.lastFetchDate,
 	}
 }

 type FeedService struct {
 	database *sqlx.DB
+	reader   *Reader
 }

-func NewFeedService(db *sqlx.DB) FeedService {
-	return FeedService{database: db}
-}
-
-func (s FeedService) AddFeed(link string, tags []string, submitter uuid.UUID) (Feed, error) {
-	feed := FeedDao{
-		Id:          uuid.New(),
-		Name:        link,
-		Link:        link,
-		FaviconUrl:  link,
-		Tags:        tags,
-		SubmitterId: submitter,
+func NewFeedService(db *sqlx.DB, reader *Reader) FeedService {
+	return FeedService{
+		database: db,
+		reader:   reader,
 	}
+}

+func (s FeedService) GetFeedData(link string) ([]Feed, error) {
+	parsed, err := s.reader.ReadFeed(link, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	return []Feed{
+		{
+			Id:         uuid.New(),
+			Name:       parsed.Title,
+			Link:       link,
+			FaviconUrl: fmt.Sprintf("%s/favicon.ico", parsed.Link),
+			AddedDate:  time.Now(),
+		},
+	}, nil
+}
+
+func (s FeedService) AddFeed(feed Feed) (Feed, error) {
 	_, err := s.database.NamedExec(
-		`insert into feeds (id, name, link, favicon_url, tags, submitter_id, added_date)
-		values (:id, :name, :link, :favicon_url, :tags, :submitter_id, :added_date)`,
-		feed,
+		`insert into feeds (id, name, link, favicon_url, tags, submitter_id, added_date, etag, last_fetch_date)
+		values (:id, :name, :link, :favicon_url, :tags, :submitter_id, :added_date, :etag, :last_fetch_date)`,
+		feed.ToDao(),
 	)
 	if err != nil {
 		return Feed{}, err
 	}
-	return feed.ToFeed(), nil
+	return feed, nil
 }

 func (s FeedService) ListFeeds() ([]Feed, error) {
 	ret := []FeedDao{}
 	err := s.database.Select(
 		&ret,
-		`select f.*, u.id as "submitter.id", u.name as "submitter.name", u.email as "submitter.email", u.password as "submitter.password" from feeds
-		as f left join users as u on u.id = f.submitter_id
+		`select f.*, u.id as "submitter.id", u.name as "submitter.name", u.email as "submitter.email", u.password as "submitter.password"
+		from feeds as f left
+		join users as u on u.id = f.submitter_id
 		order by added_date`,
 	)
 	if err != nil {
@@ -85,3 +121,29 @@ func (s FeedService) ListFeeds() ([]Feed, error) {
 	}
 	return Map(ret, func(f FeedDao, _ int) Feed { return f.ToFeed() }), nil
 }
+
+func (s FeedService) UpdateSyncStatus(id uuid.UUID, etag string, lastFetchDate *time.Time) error {
+	_, err := s.database.NamedExec(
+		`update feeds set etag = :etag, last_fetch_date = :date, sync_error = :err
+		where id = :id`,
+		map[string]interface{}{
+			"id":   id,
+			"etag": etag,
+			"date": lastFetchDate,
+			"err":  nil,
+		},
+	)
+	return err
+}
+
+func (s FeedService) SaveSyncError(id uuid.UUID, error error) error {
+	_, err := s.database.NamedExec(
+		`update feeds set sync_error = :err
+		where id = :i`,
+		map[string]interface{}{
+			"id":  id,
+			"err": error.Error(),
+		},
+	)
+	return err
+}
@@ -10,18 +10,25 @@ require (
 	github.com/labstack/echo-jwt/v4 v4.2.0
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/lib/pq v1.10.9
+	github.com/mmcdole/gofeed v1.3.0
 	golang.org/x/crypto v0.22.0
 )

 require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	golang.org/x/net v0.24.0 // indirect
@@ -1,5 +1,10 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
@@ -18,10 +23,13 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
 github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/labstack/echo-jwt/v4 v4.2.0 h1:odSISV9JgcSCuhgQSV/6Io3i7nUmfM/QkBeR5GVJj5c=
 github.com/labstack/echo-jwt/v4 v4.2.0/go.mod h1:MA2RqdXdEn4/uEglx0HcUOgQSyBaTh5JcaHIan3biwU=
 github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
@@ -39,8 +47,19 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -49,15 +68,21 @@ github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQ
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -0,0 +1,94 @@
+package vex
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+)
+
+type Reader struct {
+	feedReader *gofeed.Parser
+	client     *http.Client
+}
+
+func NewRssReader(client *http.Client) Reader {
+	return Reader{
+		feedReader: gofeed.NewParser(),
+		client:     client,
+	}
+}
+
+type GoFeed struct {
+	*gofeed.Feed
+
+	ETag         string
+	LastModified time.Time
+}
+
+var gmt, _ = time.LoadLocation("GMT")
+
+func (r *Reader) ReadFeed(url string, etag string, lastModified *time.Time) (*GoFeed, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Gofeed/1.0")
+
+	if etag != "" {
+		req.Header.Set("If-None-Match", etag)
+	}
+	if lastModified != nil {
+		req.Header.Set("If-Modified-Since", lastModified.In(gmt).Format(time.RFC1123))
+	}
+
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp != nil {
+		defer func() {
+			ce := resp.Body.Close()
+			if ce != nil {
+				err = ce
+			}
+		}()
+	}
+
+	if resp.StatusCode == http.StatusNotModified {
+		return nil, nil
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, gofeed.HTTPError{
+			StatusCode: resp.StatusCode,
+			Status:     resp.Status,
+		}
+	}
+
+	feed := &GoFeed{}
+
+	feedBody, err := r.feedReader.Parse(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	feed.Feed = feedBody
+
+	if eTag := resp.Header.Get("Etag"); eTag != "" {
+		feed.ETag = eTag
+	}
+
+	if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
+		parsed, err := time.ParseInLocation(time.RFC1123, lastModified, gmt)
+		if err == nil {
+			feed.LastModified = parsed
+		}
+	}
+
+	return feed, nil
+}
@@ -0,0 +1,82 @@
+package vex
+
+import (
+	"cmp"
+	"log"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/mmcdole/gofeed"
+)
+
+type SyncService struct {
+	reader  *Reader
+	feeds   *FeedService
+	entries *EntryService
+}
+
+func NewSyncService(reader *Reader, feeds *FeedService, entries *EntryService) SyncService {
+	return SyncService{
+		reader:  reader,
+		feeds:   feeds,
+		entries: entries,
+	}
+}
+
+func (s SyncService) SyncFeed(feed Feed) error {
+	info, err := s.reader.ReadFeed(feed.Link, feed.etag, feed.lastFetchDate)
+	if err != nil {
+		return err
+	}
+	if info == nil {
+		log.Printf("Feed %v is uptodate", feed.Link)
+		return nil
+	}
+	log.Printf("Adding %v new entries from %v", len(info.Items), feed.Link)
+	entries := Map(info.Items, func(item *gofeed.Item, _ int) EntryDao {
+		var date time.Time
+		if item.PublishedParsed != nil {
+			date = *item.PublishedParsed
+		} else {
+			date = time.Now()
+		}
+
+		return EntryDao{
+			Id:      uuid.New(),
+			Title:   item.Title,
+			Link:    item.Link,
+			Date:    date,
+			Authors: Map(item.Authors, func(author *gofeed.Person, _ int) string { return author.Name }),
+			Content: cmp.Or(item.Content, item.Description),
+			FeedId:  feed.Id,
+		}
+	})
+	err = s.entries.Add(entries)
+	if err != nil {
+		return err
+	}
+	return s.feeds.UpdateSyncStatus(feed.Id, info.ETag, &info.LastModified)
+}
+
+func (s SyncService) SyncFeeds() error {
+	feeds, err := s.feeds.ListFeeds()
+	if err != nil {
+		log.Printf("Could not retrive feeds: %v", err)
+		return err
+	}
+	for _, feed := range feeds {
+		err := s.SyncFeed(feed)
+		if err != nil {
+			log.Printf("Could not sync feed %v: %v", feed.Link, err)
+			s.feeds.SaveSyncError(feed.Id, err)
+		}
+	}
+	return nil
+}
+
+func (s SyncService) SyncFeedsForever() {
+	for {
+		s.SyncFeeds()
+		time.Sleep(15 * time.Minute)
+	}
+}
@@ -19,6 +19,8 @@ services:
    restart: on-failure
    env_file:
      - ./.env
+    ports:
+      - "5432:5432"
    volumes:
      - db:/var/lib/postgresql/data
      - ./sql/create.sql:/docker-entrypoint-initdb.d/init.sql
@@ -4,5 +4,6 @@
      go
      wgo
      pgformatter
+      postgresql
    ];
  }
@@ -13,6 +13,9 @@ create table if not exists feeds(
 	tags text[] not null,
 	submitter_id uuid not null references users(id),
 	added_date timestamp with time zone not null,
+	etag text,
+	last_fetch_date timestamp with time zone,
+	sync_error text
 );

 create table if not exists entries(
@@ -22,7 +25,7 @@ create table if not exists entries(
 	link text not null,
 	date timestamp with time zone not null,
 	content text not null,
-	author text
+	authors text[] not null
 );

 create table if not exists entries_users(
@@ -32,6 +35,6 @@ create table if not exists entries_users(
 	is_bookmarked bool not null,
 	is_read_later bool not null,
 	is_ignored bool not null,
-	constraint entries_users_pk primary key(user_id, feed_id)
+	constraint entries_users_pk primary key (user_id, feed_id)
 );