From ec6d61b221604863adecf3aec1196b3335b32198 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Tue, 21 Jun 2022 11:35:52 +0200 Subject: [PATCH] rss endpoint implementation as suggested by Chris --- INSTALL.md | 11 ++--- README.md | 12 ++++++ app/mf/microformats.go | 12 ++++-- app/routes.go | 9 ++-- app/rss/handler.go | 92 +++++++++++++++++++++++++++++++++++++++++ app/rss/handler_test.go | 75 +++++++++++++++++++++++++++++++++ app/rss/mentionsrss.xml | 38 +++++++++++++++++ app/server.go | 4 +- app/server_test.go | 8 ++-- common/time.go | 14 ++++--- common/time_test.go | 4 +- db/buntrepo_test.go | 3 +- 12 files changed, 255 insertions(+), 27 deletions(-) create mode 100644 app/rss/handler.go create mode 100644 app/rss/handler_test.go create mode 100644 app/rss/mentionsrss.xml diff --git a/INSTALL.md b/INSTALL.md index 035a857..86d9705 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,11 +32,12 @@ Place a `config.json` file in the same directory that looks like this: (below ar } ``` -- baseURL, with trailing slash: base access point, used in approval/admin panel -- adminEmail, the e-mail address to send notificaions to. If absent, will not send out mails. **uses 127.0.0.1:25 postfix** at the moment. -- port, host: http server params -- token, allowedWebmentionSources: see below, used for authentication -- blacklist/whitelist: domains from which we do (NOT) send to or accept mentions from. +- `baseURL`, with trailing slash: base access point, used in approval/admin panel +- `adminEmail`, the e-mail address to send notificaions to. If absent, will not send out mails. **uses 127.0.0.1:25 postfix** at the moment. +- `port`, host: http server params +- `token`: see below, used for authentication +- `blacklist`/`whitelist`: domains from which we do (NOT) send to or accept mentions from. This is usually the domain of the `source` in the receiving mention. **Note**: the blacklist is also used to block outgoing mentions. +- `allowedWebmentionSources`: your own domains which go-jamming is able to receive mentions from. This is usually domain of the `target` in the receiving mention. If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`. diff --git a/README.md b/README.md index 3fb67c9..86562e6 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,18 @@ Will result in a `200 OK` - that returns XML according to [The W3 pingback XML-R Happens automatically through `PUT /webmention/:domain/:token`! Links that are discovered as `rel="pingback"` that **do not** already have a webmention link will be processed as XML-RPC requests to be send. +### 3. Keep track of mentions + +#### 3.1 `GET /feed/[domain]/[token]` + +To keep track of all incoming mentions, simply subscribe to the above URL for each of your domains using your favorite RSS reader. + +Mentions to approve also appear in the feed, together with links to accept or reject them with a single click! (See below, section "mentions in moderation"). + +#### 3.2 SMTP `localhost` + +Messages of incoming mentions can also be sent via e-mail if you have a localhost SMTP server setup. See the `INSTALL.md` for more instructions and config parameters on how to do so. + --- ## Troubleshooting diff --git a/app/mf/microformats.go b/app/mf/microformats.go index 96405c1..b9c3e1e 100644 --- a/app/mf/microformats.go +++ b/app/mf/microformats.go @@ -9,7 +9,7 @@ import ( ) const ( - dateFormatWithTimeZone = "2006-01-02T15:04:05-07:00" + DateFormatWithTimeZone = "2006-01-02T15:04:05-07:00" dateFormatWithAbsoluteTimeZone = "2006-01-02T15:04:05-0700" dateFormatWithTimeZoneSuffixed = "2006-01-02T15:04:05.000Z" dateFormatWithoutTimeZone = "2006-01-02T15:04:05" @@ -22,7 +22,7 @@ var ( // This is similar to Hugo's string-to-date casting system // See https://github.com/spf13/cast/blob/master/caste.go supportedFormats = []string{ - dateFormatWithTimeZone, + DateFormatWithTimeZone, dateFormatWithAbsoluteTimeZone, dateFormatWithTimeZoneSuffixed, dateFormatWithSecondsWithoutTimeZone, @@ -81,6 +81,10 @@ type IndiewebData struct { Target string `json:"target"` } +func (id *IndiewebData) PublishedDate() time.Time { + return common.ToTime(id.Published, DateFormatWithTimeZone) +} + func (id *IndiewebData) AsMention() Mention { return Mention{ Source: id.Source, @@ -93,7 +97,7 @@ func (id *IndiewebData) IsEmpty() bool { } func PublishedNow() string { - return common.Now().UTC().Format(dateFormatWithTimeZone) + return common.Now().UTC().Format(DateFormatWithTimeZone) } func shorten(txt string) string { @@ -185,7 +189,7 @@ func Published(hEntry *microformats.Microformat) string { if err != nil { continue } - return formatted.Format(dateFormatWithTimeZone) + return formatted.Format(DateFormatWithTimeZone) } return PublishedNow() diff --git a/app/routes.go b/app/routes.go index 2d15ae6..6753928 100644 --- a/app/routes.go +++ b/app/routes.go @@ -5,6 +5,7 @@ import ( "brainbaking.com/go-jamming/app/index" "brainbaking.com/go-jamming/app/pictures" "brainbaking.com/go-jamming/app/pingback" + "brainbaking.com/go-jamming/app/rss" "brainbaking.com/go-jamming/app/webmention" ) @@ -20,12 +21,14 @@ func (s *server) routes() { s.router.HandleFunc("/pingback", pingback.HandlePost(c, db)).Methods("POST") s.router.HandleFunc("/webmention", webmention.HandlePost(c, db)).Methods("POST") + s.router.HandleFunc("/feed/{domain}/{token}", s.domainAndTokenOnly(rss.HandleGet(c, db))).Methods("GET") + s.router.HandleFunc("/webmention/{domain}/{token}", s.domainAndTokenOnly(webmention.HandleGet(db))).Methods("GET") s.router.HandleFunc("/webmention/{domain}/{token}", s.domainAndTokenOnly(webmention.HandlePut(c, db))).Methods("PUT") s.router.HandleFunc("/webmention/{domain}/{token}", s.domainAndTokenOnly(webmention.HandleDelete(db))).Methods("DELETE") - s.router.HandleFunc("/admin/{token}", s.authorizedOnly(admin.HandleGet(c, db))).Methods("GET") + s.router.HandleFunc("/admin/{token}", s.tokenOnly(admin.HandleGet(c, db))).Methods("GET") s.router.HandleFunc("/admin/{domain}/{token}", s.domainAndTokenOnly(admin.HandleGetToApprove(db))).Methods("GET") - s.router.HandleFunc("/admin/approve/{token}/{key}", s.authorizedOnly(admin.HandleApprove(c, db))).Methods("GET") - s.router.HandleFunc("/admin/reject/{token}/{key}", s.authorizedOnly(admin.HandleReject(c, db))).Methods("GET") + s.router.HandleFunc("/admin/approve/{token}/{key}", s.tokenOnly(admin.HandleApprove(c, db))).Methods("GET") + s.router.HandleFunc("/admin/reject/{token}/{key}", s.tokenOnly(admin.HandleReject(c, db))).Methods("GET") } diff --git a/app/rss/handler.go b/app/rss/handler.go new file mode 100644 index 0000000..fef7bc5 --- /dev/null +++ b/app/rss/handler.go @@ -0,0 +1,92 @@ +package rss + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "brainbaking.com/go-jamming/db" + "fmt" + "github.com/gorilla/mux" + "github.com/rs/zerolog/log" + "net/http" + "sort" + "text/template" + "time" +) + +import _ "embed" + +const ( + MaxRssItems = 50 +) + +//go:embed mentionsrss.xml +var mentionsrssTemplate []byte + +type RssMentions struct { + Domain string + Date time.Time + Items []*RssMentionItem +} + +type RssMentionItem struct { + ApproveURL string + RejectURL string + Data *mf.IndiewebData +} + +func asTemplate(name string, data []byte) *template.Template { + tmpl, err := template.New(name).Parse(string(data)) + if err != nil { + log.Fatal().Err(err).Str("name", name).Msg("Template invalid") + } + return tmpl +} + +func HandleGet(c *common.Config, repo db.MentionRepo) http.HandlerFunc { + tmpl := asTemplate("mentionsRss", mentionsrssTemplate) + + return func(w http.ResponseWriter, r *http.Request) { + domain := mux.Vars(r)["domain"] + + mentions := getLatestMentions(domain, repo, c) + err := tmpl.Execute(w, RssMentions{ + Items: mentions, + Date: time.Now(), + Domain: domain, + }) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + log.Error().Err(err).Msg("Unable to fill in dashboard template") + } + } +} + +func getLatestMentions(domain string, repo db.MentionRepo, c *common.Config) []*RssMentionItem { + toMod := repo.GetAllToModerate(domain).Data + all := repo.GetAll(domain).Data + + var data []*RssMentionItem + for _, v := range toMod { + wm := v.AsMention() + data = append(data, &RssMentionItem{ + Data: v, + ApproveURL: fmt.Sprintf("%sadmin/approve/%s/%s", c.BaseURL, c.Token, wm.Key()), + RejectURL: fmt.Sprintf("%sadmin/reject/%s/%s", c.BaseURL, c.Token, wm.Key()), + }) + } + for _, v := range all { + data = append(data, &RssMentionItem{ + Data: v, + }) + } + + // TODO this date is the published date, not the webmention received date! + // This means it "might" disappear after the cutoff point in the RSS feed, and we don't store a received timestamp + sort.Slice(data, func(i, j int) bool { + return data[i].Data.PublishedDate().After(data[j].Data.PublishedDate()) + }) + if len(data) > MaxRssItems { + return data[0:MaxRssItems] + } + return data +} diff --git a/app/rss/handler_test.go b/app/rss/handler_test.go new file mode 100644 index 0000000..1377177 --- /dev/null +++ b/app/rss/handler_test.go @@ -0,0 +1,75 @@ +package rss + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "brainbaking.com/go-jamming/db" + "fmt" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" +) + +var ( + cnf = &common.Config{ + BaseURL: "http://localhost:1337/", + Port: 1337, + Token: "miauwkes", + AllowedWebmentionSources: []string{"brainbaking.com"}, + Blacklist: []string{}, + Whitelist: []string{"brainbaking.com"}, + } + repo db.MentionRepo +) + +func init() { + repo = db.NewMentionRepo(cnf) +} +func TestHandleGet(t *testing.T) { + wmInMod := mf.Mention{ + Source: "https://infos.by/markdown-v-nauke/", + Target: "https://brainbaking.com/post/2021/02/writing-academic-papers-in-markdown/", + } + wmApproved := mf.Mention{ + Source: "https://brainbaking.com/post/2022/04/equality-in-game-credits/", + Target: "https://brainbaking.com/", + } + + repo.InModeration(wmInMod, &mf.IndiewebData{ + Source: wmInMod.Source, + Target: wmInMod.Target, + Name: "inmod1", + }) + repo.Save(wmApproved, &mf.IndiewebData{ + Source: wmApproved.Source, + Target: wmApproved.Target, + Name: "approved1", + }) + r := mux.NewRouter() + r.HandleFunc("/feed/{domain}/{token}", HandleGet(cnf, repo)).Methods("GET") + ts := httptest.NewServer(r) + + t.Cleanup(func() { + os.Remove("config.json") + ts.Close() + db.Purge() + }) + + client := &http.Client{} + req, err := http.NewRequest("GET", fmt.Sprintf("%s/feed/%s/%s", ts.URL, cnf.AllowedWebmentionSources[0], cnf.Token), nil) + + resp, err := client.Do(req) + assert.NoError(t, err) + + contentBytes, _ := ioutil.ReadAll(resp.Body) + content := string(contentBytes) + defer resp.Body.Close() + + assert.Contains(t, content, "Go-Jamming @ brainbaking.com") + assert.Contains(t, content, "To Moderate: inmod1 ()") + assert.Contains(t, content, "approved1 ()") +} diff --git a/app/rss/mentionsrss.xml b/app/rss/mentionsrss.xml new file mode 100644 index 0000000..fefd189 --- /dev/null +++ b/app/rss/mentionsrss.xml @@ -0,0 +1,38 @@ + + + + Go-Jamming @ {{ .Domain }} + Go-Jamming @ {{ .Domain }} + Go-Jamming + en-us + {{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" }} + + {{ range .Items }} + + {{ if .ApproveURL }}To Moderate: {{ end }}{{ .Data.Name | html }} ({{ .Data.Url }}) + {{ .Data.Target }} + {{ .Data.PublishedDate.Format "Mon, 02 Jan 2006 15:04:05 -0700" }} + {{ .Data.Author.Name | html }} + + ✅ Approve this mention!
+ ❌ Reject this mention!

+ {{ end }} + + Author: {{ .Data.Author }}
+ Name: {{ .Data.Name }}
+ Published: {{ .Data.Published }}
+ Type: {{ .Data.IndiewebType }}
+ Url: {{ .Data.Url }}

+ + Source: {{ .Data.Source }}
+ Target: {{ .Data.Target }}

+ + Content: {{ .Data.Content }} + ]]> + + + {{ end }} + + \ No newline at end of file diff --git a/app/server.go b/app/server.go index daaf60c..040b5e7 100644 --- a/app/server.go +++ b/app/server.go @@ -21,7 +21,7 @@ type server struct { } func (s *server) domainAndTokenOnly(h http.HandlerFunc) http.HandlerFunc { - return s.domainOnly(s.authorizedOnly(h)) + return s.domainOnly(s.tokenOnly(h)) } func (s *server) domainOnly(h http.HandlerFunc) http.HandlerFunc { @@ -35,7 +35,7 @@ func (s *server) domainOnly(h http.HandlerFunc) http.HandlerFunc { } } -func (s *server) authorizedOnly(h http.HandlerFunc) http.HandlerFunc { +func (s *server) tokenOnly(h http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) if vars["token"] != s.conf.Token { diff --git a/app/server_test.go b/app/server_test.go index e70f07b..48b1d77 100644 --- a/app/server_test.go +++ b/app/server_test.go @@ -14,13 +14,13 @@ var conf = &common.Config{ AllowedWebmentionSources: []string{"http://ewelja.be"}, } -func TestAuthorizedOnlyUnauthorizedWithWrongToken(t *testing.T) { +func TestTokenOnlyUnauthorizedWithWrongToken(t *testing.T) { srv := &server{ conf: conf, } passed := false - handler := srv.authorizedOnly(func(writer http.ResponseWriter, request *http.Request) { + handler := srv.tokenOnly(func(writer http.ResponseWriter, request *http.Request) { passed = true }) r, _ := http.NewRequest("PUT", "/whatever", nil) @@ -54,13 +54,13 @@ func TestDomainOnlyWithWrongDomain(t *testing.T) { assert.False(t, passed, "should not have called unauthorized func") } -func TestAuthorizedOnlyOkIfTokenAndDomainMatch(t *testing.T) { +func TestTokenOnlyOkIfTokenAndDomainMatch(t *testing.T) { srv := &server{ conf: conf, } passed := false - handler := srv.authorizedOnly(func(writer http.ResponseWriter, request *http.Request) { + handler := srv.tokenOnly(func(writer http.ResponseWriter, request *http.Request) { passed = true }) r, _ := http.NewRequest("PUT", "/whatever", nil) diff --git a/common/time.go b/common/time.go index 7b8d249..da55c2a 100644 --- a/common/time.go +++ b/common/time.go @@ -1,7 +1,6 @@ package common import ( - "github.com/rs/zerolog/log" "time" ) @@ -20,14 +19,17 @@ func TimeToIso(theTime time.Time) string { // IsoToTime converts an ISO time string into a time.Time object // As produced by clients using day.js - e.g. 2021-04-09T15:51:43.732Z -func IsoToTime(since string) time.Time { - if since == "" { +func IsoToTime(date string) time.Time { + return ToTime(date, IsoFormat) +} + +func ToTime(date string, format string) time.Time { + if date == "" { return time.Time{} } - t, err := time.Parse(IsoFormat, since) + t, err := time.Parse(format, date) if err != nil { - log.Warn().Str("time", since).Msg("Invalid ISO date, reverting to now()") - return Now() + return time.Time{} } return t } diff --git a/common/time_test.go b/common/time_test.go index 199ff0a..e1842db 100644 --- a/common/time_test.go +++ b/common/time_test.go @@ -42,9 +42,9 @@ func (s *TimeSuite) TestIsoToTimeInISOString() { assert.Equal(s.T(), expectedtime.Second(), since.Second()) } -func (s *TimeSuite) TestIsoToTimeInvalidStringReturnsNowTime() { +func (s *TimeSuite) TestIsoToTimeInvalidStringReturnsZeroTime() { since := IsoToTime("woef ik ben een hondje") - assert.Equal(s.T(), s.nowtime, since) + assert.True(s.T(), since.IsZero()) } func (s *TimeSuite) TestIsoToTimeEmptyReturnsZeroTime() { diff --git a/db/buntrepo_test.go b/db/buntrepo_test.go index de13c79..4a893fe 100644 --- a/db/buntrepo_test.go +++ b/db/buntrepo_test.go @@ -132,7 +132,8 @@ func TestGetAllAndSaveSomeJson(t *testing.T) { db.Save(mf.Mention{ Target: "https://pussycat.com/coolpussy.html", }, &mf.IndiewebData{ - Name: "lolz", + Name: "lolz", + Published: "2021-07-24T23:27:25+01:00", }) results := db.GetAll("pussycat.com")