diff --git a/app/admin/handler.go b/app/admin/handler.go new file mode 100644 index 0000000..bb06d08 --- /dev/null +++ b/app/admin/handler.go @@ -0,0 +1,47 @@ +package admin + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "brainbaking.com/go-jamming/db" + "brainbaking.com/go-jamming/rest" + "github.com/gorilla/mux" + "net/http" +) + +func HandleGet(repo db.MentionRepo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + domain := mux.Vars(r)["domain"] + rest.Json(w, repo.GetAllToModerate(domain)) + } +} + +// TODO validate or not? see webmention.HandlePost +// TODO unit tests +func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + wm := mf.Mention{ + Source: r.FormValue("source"), + Target: r.FormValue("target"), + } + + repo.Approve(wm) + c.AddToWhitelist(wm.SourceDomain()) + w.WriteHeader(200) + } +} + +func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + wm := mf.Mention{ + Source: r.FormValue("source"), + Target: r.FormValue("target"), + } + + repo.Reject(wm) + c.AddToBlacklist(wm.SourceDomain()) + w.WriteHeader(200) + } +} diff --git a/app/mf/mention.go b/app/mf/mention.go index 9e1a0a8..9141288 100644 --- a/app/mf/mention.go +++ b/app/mf/mention.go @@ -24,13 +24,18 @@ func (wm Mention) String() string { return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target) } -// Domain parses the target url to extract the domain as part of the allowed webmention targets. +// TargetDomain parses the target url to extract the domain as part of the allowed webmention targets. // This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling. // Assumes http(s) protocol, which should have been validated by now. -func (wm Mention) Domain() string { +func (wm Mention) TargetDomain() string { return rest.Domain(wm.Target) } +// SoureceDomain converts the Source to a domain name to be used in whitelisting/blacklisting (See TargetDomain()). +func (wm Mention) SourceDomain() string { + return rest.Domain(wm.Source) +} + // Key returns a unique string representation of the mention for use in storage. // TODO Profiling indicated that md5() consumes a lot of CPU power, so this could be replaced with db migration. func (wm Mention) Key() string { diff --git a/app/mf/mention_test.go b/app/mf/mention_test.go index a6b6be0..0e67cdd 100644 --- a/app/mf/mention_test.go +++ b/app/mf/mention_test.go @@ -5,11 +5,21 @@ import ( "testing" ) -func TestDomainParseFromTarget(t *testing.T) { +func TestTargetDomainDomain(t *testing.T) { wm := Mention{ Source: "source", Target: "http://patat.be/frietjes/zijn/lekker", } - assert.Equal(t, "patat.be", wm.Domain()) + assert.Equal(t, "patat.be", wm.TargetDomain()) +} + +func TestSourceDomain(t *testing.T) { + wm := Mention{ + Source: "http://patat.be/frietjes/zijn/lekker", + Target: "source", + } + + assert.Equal(t, "patat.be", wm.SourceDomain()) + } diff --git a/app/routes.go b/app/routes.go index 1000260..861a2ee 100644 --- a/app/routes.go +++ b/app/routes.go @@ -1,6 +1,7 @@ package app import ( + "brainbaking.com/go-jamming/app/admin" "brainbaking.com/go-jamming/app/index" "brainbaking.com/go-jamming/app/pictures" "brainbaking.com/go-jamming/app/pingback" @@ -18,7 +19,12 @@ func (s *server) routes() { s.router.HandleFunc("/pictures/{picture}", pictures.Handle(db)).Methods("GET") s.router.HandleFunc("/pingback", pingback.HandlePost(c, db)).Methods("POST") s.router.HandleFunc("/webmention", webmention.HandlePost(c, db)).Methods("POST") + s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleGet(db))).Methods("GET") s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandlePut(c, db))).Methods("PUT") s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleDelete(db))).Methods("DELETE") + + s.router.HandleFunc("/admin/{domain}/{token}", s.authorizedOnly(admin.HandleGet(db))).Methods("GET") + s.router.HandleFunc("/admin/approve/{token}", s.authorizedOnly(admin.HandleApprove(c, db))).Methods("POST") + s.router.HandleFunc("/admin/reject/{token}", s.authorizedOnly(admin.HandleReject(c, db))).Methods("POST") } diff --git a/app/webmention/recv/receive.go b/app/webmention/recv/receive.go index 3c3e429..d491d88 100644 --- a/app/webmention/recv/receive.go +++ b/app/webmention/recv/receive.go @@ -62,11 +62,23 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) { indieweb := recv.convertBodyToIndiewebData(body, wm, data) recv.processAuthorPicture(indieweb) - key, err := recv.Repo.Save(wm, indieweb) - if err != nil { - log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db") + recv.saveMentionToDatabase(wm, indieweb) +} + +func (recv *Receiver) saveMentionToDatabase(wm mf.Mention, indieweb *mf.IndiewebData) { + if recv.Conf.IsWhitelisted(wm.Source) { + key, err := recv.Repo.Save(wm, indieweb) + if err != nil { + log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db") + } + log.Info().Str("key", key).Msg("OK: Webmention processed, in whitelist.") + } else { + key, err := recv.Repo.InModeration(wm, indieweb) + if err != nil { + log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to in moderation db") + } + log.Info().Str("key", key).Msg("OK: Webmention processed, in moderation.") } - log.Info().Str("key", key).Msg("OK: Webmention processed.") } func (recv *Receiver) processAuthorPicture(indieweb *mf.IndiewebData) { diff --git a/app/webmention/recv/receive_test.go b/app/webmention/recv/receive_test.go index 1f96193..cce3ed1 100644 --- a/app/webmention/recv/receive_test.go +++ b/app/webmention/recv/receive_test.go @@ -24,6 +24,10 @@ var conf = &common.Config{ Blacklist: []string{ "blacklisted.com", }, + Whitelist: []string{ + "brainbaking.com", + "jefklakscodex.com", + }, } func TestSaveAuthorPictureLocally(t *testing.T) { @@ -233,6 +237,33 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi assert.Empty(t, indb) } +func TestReceiveFromNotInWhitelistSavesInModeration(t *testing.T) { + wm := mf.Mention{ + Source: "https://brainbaking.com/valid-indieweb-source.html", + Target: "https://brainbaking.com/valid-indieweb-target.html", + } + cnf := &common.Config{ + AllowedWebmentionSources: []string{ + "brainbaking.com", + }, + Blacklist: []string{}, + Whitelist: []string{}, + } + repo := db.NewMentionRepo(cnf) + t.Cleanup(db.Purge) + receiver := &Receiver{ + Conf: cnf, + Repo: repo, + RestClient: &mocks.RestClientMock{ + GetBodyFunc: mocks.RelPathGetBodyFunc("../../../mocks/"), + }, + } + + receiver.Receive(wm) + assert.Empty(t, repo.GetAll("brainbaking.com").Data) + assert.Equal(t, 1, len(repo.GetAllToModerate("brainbaking.com").Data)) +} + func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) { wm := mf.Mention{ Source: "https://blacklisted.com/whoops", @@ -248,6 +279,7 @@ func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) { receiver.Receive(wm) assert.Empty(t, repo.GetAll("brainbaking.com").Data) + assert.Empty(t, repo.GetAllToModerate("brainbaking.com").Data) } func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) { @@ -268,6 +300,7 @@ func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) { receiver.Receive(wm) assert.Empty(t, repo.GetAll("brainbaking.com").Data) + assert.Empty(t, repo.GetAllToModerate("brainbaking.com").Data) } func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *testing.T) { @@ -275,10 +308,19 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t * Source: "https://brid.gy/post/twitter/ChrisAldrich/1387130900962443264", Target: "https://brainbaking.com/", } - repo := db.NewMentionRepo(conf) + cnf := &common.Config{ + AllowedWebmentionSources: []string{ + "brainbaking.com", + }, + Whitelist: []string{ + "brid.gy", + }, + } + + repo := db.NewMentionRepo(cnf) t.Cleanup(db.Purge) recv := &Receiver{ - Conf: conf, + Conf: cnf, Repo: repo, } diff --git a/common/config.go b/common/config.go index d9f0f13..df4f711 100644 --- a/common/config.go +++ b/common/config.go @@ -18,14 +18,23 @@ type Config struct { DataPath string `json:"dataPath"` AllowedWebmentionSources []string `json:"allowedWebmentionSources"` Blacklist []string `json:"blacklist"` + Whitelist []string `json:"whitelist"` } func (c *Config) IsBlacklisted(url string) bool { + return isListedIn(url, c.Blacklist) +} + +func (c *Config) IsWhitelisted(url string) bool { + return isListedIn(url, c.Whitelist) +} + +func isListedIn(url string, list []string) bool { if !strings.HasPrefix(url, "http") { return false } domain := rest.Domain(url) - return Includes(c.Blacklist, domain) + return Includes(list, domain) } func (c *Config) Zone() *time.Location { @@ -67,14 +76,26 @@ func Configure() *Config { return c } +// AddToBlacklist adds the given domain to the blacklist slice and persists to disk. func (c *Config) AddToBlacklist(domain string) { - for _, d := range c.Blacklist { - if d == domain { - return + c.Blacklist = addToList(domain, c.Blacklist) + c.Save() +} + +// AddToWhitelist adds the given domain to the whitelist slice and persists to disk. +func (c *Config) AddToWhitelist(domain string) { + c.Whitelist = addToList(domain, c.Whitelist) + c.Save() +} + +func addToList(key string, arr []string) []string { + for _, d := range arr { + if d == key { + return arr } } - c.Blacklist = append(c.Blacklist, domain) + return append(arr, key) } func (c *Config) Save() { @@ -113,5 +134,6 @@ func defaultConfig() *Config { UtcOffset: 60, AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"}, Blacklist: []string{"youtube.com"}, + Whitelist: []string{"brainbaking.com"}, } } diff --git a/common/config_test.go b/common/config_test.go index fa80b03..e0bd01e 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -52,16 +52,48 @@ func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) { assert.Contains(t, newConfig.Blacklist, "somethingnew.be") } -func TestAddToBlacklistNotYetAddsToList(t *testing.T) { +func TestWhitelist(t *testing.T) { + conf := Config{ + Whitelist: []string{ + "youtube.com", + }, + Port: 123, + Token: "token", + AllowedWebmentionSources: []string{"blah.com"}, + } + t.Cleanup(func() { + os.Remove("config.json") + }) + + conf.AddToWhitelist("dinges.be") + assert.Contains(t, conf.Whitelist, "dinges.be") + assert.Equal(t, 2, len(conf.Whitelist)) + + confFromFile := Configure() + assert.Contains(t, confFromFile.Whitelist, "dinges.be") + assert.Equal(t, 2, len(confFromFile.Whitelist)) +} + +func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) { conf := Config{ Blacklist: []string{ "youtube.com", }, + Port: 123, + Token: "token", + AllowedWebmentionSources: []string{"blah.com"}, } + t.Cleanup(func() { + os.Remove("config.json") + }) conf.AddToBlacklist("dinges.be") assert.Contains(t, conf.Blacklist, "dinges.be") assert.Equal(t, 2, len(conf.Blacklist)) + + confFromFile := Configure() + assert.Contains(t, confFromFile.Blacklist, "dinges.be") + assert.Equal(t, 2, len(confFromFile.Blacklist)) } func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) { @@ -69,13 +101,59 @@ func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) { Blacklist: []string{ "youtube.com", }, + Port: 123, + Token: "token", + AllowedWebmentionSources: []string{"blah.com"}, } + t.Cleanup(func() { + os.Remove("config.json") + }) conf.AddToBlacklist("youtube.com") assert.Contains(t, conf.Blacklist, "youtube.com") assert.Equal(t, 1, len(conf.Blacklist)) } +func TestIsWhitelisted(t *testing.T) { + cases := []struct { + label string + url string + expected bool + }{ + { + "do not whitelist if domain is part of relative url", + "https://brainbaking.com/post/youtube.com-sucks", + false, + }, + { + "whitelist if https domain is on the list", + "https://youtube.com/stuff", + true, + }, + { + "whitelist if http domain is on the list", + "http://youtube.com/stuff", + true, + }, + { + "do not whitelist if relative url", + "/youtube.com", + false, + }, + } + + conf := Config{ + Whitelist: []string{ + "youtube.com", + }, + } + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + assert.Equal(t, tc.expected, conf.IsWhitelisted(tc.url)) + }) + } +} + func TestIsBlacklisted(t *testing.T) { cases := []struct { label string diff --git a/db/buntrepo.go b/db/buntrepo.go index 6593b66..34f773d 100644 --- a/db/buntrepo.go +++ b/db/buntrepo.go @@ -104,14 +104,17 @@ func (r *mentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er } func (r *mentionRepoBunt) mentionToKey(wm mf.Mention) string { - return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain()) + return fmt.Sprintf("%s:%s", wm.Key(), wm.TargetDomain()) } // Get returns a single unmarshalled json value based on the mention key. // It returns the unmarshalled result or nil if something went wrong. func (r *mentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData { + return r.getByKey(r.mentionToKey(wm)) +} + +func (r *mentionRepoBunt) getByKey(key string) *mf.IndiewebData { var data mf.IndiewebData - key := r.mentionToKey(wm) err := r.db.View(func(tx *buntdb.Tx) error { val, err := tx.Get(key) if err != nil { diff --git a/main.go b/main.go index e8f9de7..9b50bf7 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,6 @@ func blacklistDomain(domain string) { log.Info().Str("domain", domain).Msg("Blacklisting...") config := common.Configure() config.AddToBlacklist(domain) - config.Save() repo := db.NewMentionRepo(config) for _, domain := range config.AllowedWebmentionSources {