foundations for approve/reject system, /admin router endpoints

This commit is contained in:
Wouter Groeneveld 2022-04-23 11:35:53 +02:00
parent 3ec6694757
commit 2e504eaa65
10 changed files with 243 additions and 19 deletions

47
app/admin/handler.go Normal file
View File

@ -0,0 +1,47 @@
package admin
import (
"brainbaking.com/go-jamming/app/mf"
"brainbaking.com/go-jamming/common"
"brainbaking.com/go-jamming/db"
"brainbaking.com/go-jamming/rest"
"github.com/gorilla/mux"
"net/http"
)
func HandleGet(repo db.MentionRepo) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
domain := mux.Vars(r)["domain"]
rest.Json(w, repo.GetAllToModerate(domain))
}
}
// TODO validate or not? see webmention.HandlePost
// TODO unit tests
func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
wm := mf.Mention{
Source: r.FormValue("source"),
Target: r.FormValue("target"),
}
repo.Approve(wm)
c.AddToWhitelist(wm.SourceDomain())
w.WriteHeader(200)
}
}
func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
wm := mf.Mention{
Source: r.FormValue("source"),
Target: r.FormValue("target"),
}
repo.Reject(wm)
c.AddToBlacklist(wm.SourceDomain())
w.WriteHeader(200)
}
}

View File

@ -24,13 +24,18 @@ func (wm Mention) String() string {
return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
}
// Domain parses the target url to extract the domain as part of the allowed webmention targets.
// TargetDomain parses the target url to extract the domain as part of the allowed webmention targets.
// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling.
// Assumes http(s) protocol, which should have been validated by now.
func (wm Mention) Domain() string {
func (wm Mention) TargetDomain() string {
return rest.Domain(wm.Target)
}
// SoureceDomain converts the Source to a domain name to be used in whitelisting/blacklisting (See TargetDomain()).
func (wm Mention) SourceDomain() string {
return rest.Domain(wm.Source)
}
// Key returns a unique string representation of the mention for use in storage.
// TODO Profiling indicated that md5() consumes a lot of CPU power, so this could be replaced with db migration.
func (wm Mention) Key() string {

View File

@ -5,11 +5,21 @@ import (
"testing"
)
func TestDomainParseFromTarget(t *testing.T) {
func TestTargetDomainDomain(t *testing.T) {
wm := Mention{
Source: "source",
Target: "http://patat.be/frietjes/zijn/lekker",
}
assert.Equal(t, "patat.be", wm.Domain())
assert.Equal(t, "patat.be", wm.TargetDomain())
}
func TestSourceDomain(t *testing.T) {
wm := Mention{
Source: "http://patat.be/frietjes/zijn/lekker",
Target: "source",
}
assert.Equal(t, "patat.be", wm.SourceDomain())
}

View File

@ -1,6 +1,7 @@
package app
import (
"brainbaking.com/go-jamming/app/admin"
"brainbaking.com/go-jamming/app/index"
"brainbaking.com/go-jamming/app/pictures"
"brainbaking.com/go-jamming/app/pingback"
@ -18,7 +19,12 @@ func (s *server) routes() {
s.router.HandleFunc("/pictures/{picture}", pictures.Handle(db)).Methods("GET")
s.router.HandleFunc("/pingback", pingback.HandlePost(c, db)).Methods("POST")
s.router.HandleFunc("/webmention", webmention.HandlePost(c, db)).Methods("POST")
s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleGet(db))).Methods("GET")
s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandlePut(c, db))).Methods("PUT")
s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleDelete(db))).Methods("DELETE")
s.router.HandleFunc("/admin/{domain}/{token}", s.authorizedOnly(admin.HandleGet(db))).Methods("GET")
s.router.HandleFunc("/admin/approve/{token}", s.authorizedOnly(admin.HandleApprove(c, db))).Methods("POST")
s.router.HandleFunc("/admin/reject/{token}", s.authorizedOnly(admin.HandleReject(c, db))).Methods("POST")
}

View File

@ -62,11 +62,23 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {
indieweb := recv.convertBodyToIndiewebData(body, wm, data)
recv.processAuthorPicture(indieweb)
key, err := recv.Repo.Save(wm, indieweb)
if err != nil {
log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db")
recv.saveMentionToDatabase(wm, indieweb)
}
func (recv *Receiver) saveMentionToDatabase(wm mf.Mention, indieweb *mf.IndiewebData) {
if recv.Conf.IsWhitelisted(wm.Source) {
key, err := recv.Repo.Save(wm, indieweb)
if err != nil {
log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db")
}
log.Info().Str("key", key).Msg("OK: Webmention processed, in whitelist.")
} else {
key, err := recv.Repo.InModeration(wm, indieweb)
if err != nil {
log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to in moderation db")
}
log.Info().Str("key", key).Msg("OK: Webmention processed, in moderation.")
}
log.Info().Str("key", key).Msg("OK: Webmention processed.")
}
func (recv *Receiver) processAuthorPicture(indieweb *mf.IndiewebData) {

View File

@ -24,6 +24,10 @@ var conf = &common.Config{
Blacklist: []string{
"blacklisted.com",
},
Whitelist: []string{
"brainbaking.com",
"jefklakscodex.com",
},
}
func TestSaveAuthorPictureLocally(t *testing.T) {
@ -233,6 +237,33 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi
assert.Empty(t, indb)
}
func TestReceiveFromNotInWhitelistSavesInModeration(t *testing.T) {
wm := mf.Mention{
Source: "https://brainbaking.com/valid-indieweb-source.html",
Target: "https://brainbaking.com/valid-indieweb-target.html",
}
cnf := &common.Config{
AllowedWebmentionSources: []string{
"brainbaking.com",
},
Blacklist: []string{},
Whitelist: []string{},
}
repo := db.NewMentionRepo(cnf)
t.Cleanup(db.Purge)
receiver := &Receiver{
Conf: cnf,
Repo: repo,
RestClient: &mocks.RestClientMock{
GetBodyFunc: mocks.RelPathGetBodyFunc("../../../mocks/"),
},
}
receiver.Receive(wm)
assert.Empty(t, repo.GetAll("brainbaking.com").Data)
assert.Equal(t, 1, len(repo.GetAllToModerate("brainbaking.com").Data))
}
func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) {
wm := mf.Mention{
Source: "https://blacklisted.com/whoops",
@ -248,6 +279,7 @@ func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) {
receiver.Receive(wm)
assert.Empty(t, repo.GetAll("brainbaking.com").Data)
assert.Empty(t, repo.GetAllToModerate("brainbaking.com").Data)
}
func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) {
@ -268,6 +300,7 @@ func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) {
receiver.Receive(wm)
assert.Empty(t, repo.GetAll("brainbaking.com").Data)
assert.Empty(t, repo.GetAllToModerate("brainbaking.com").Data)
}
func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *testing.T) {
@ -275,10 +308,19 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *
Source: "https://brid.gy/post/twitter/ChrisAldrich/1387130900962443264",
Target: "https://brainbaking.com/",
}
repo := db.NewMentionRepo(conf)
cnf := &common.Config{
AllowedWebmentionSources: []string{
"brainbaking.com",
},
Whitelist: []string{
"brid.gy",
},
}
repo := db.NewMentionRepo(cnf)
t.Cleanup(db.Purge)
recv := &Receiver{
Conf: conf,
Conf: cnf,
Repo: repo,
}

View File

@ -18,14 +18,23 @@ type Config struct {
DataPath string `json:"dataPath"`
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
Blacklist []string `json:"blacklist"`
Whitelist []string `json:"whitelist"`
}
func (c *Config) IsBlacklisted(url string) bool {
return isListedIn(url, c.Blacklist)
}
func (c *Config) IsWhitelisted(url string) bool {
return isListedIn(url, c.Whitelist)
}
func isListedIn(url string, list []string) bool {
if !strings.HasPrefix(url, "http") {
return false
}
domain := rest.Domain(url)
return Includes(c.Blacklist, domain)
return Includes(list, domain)
}
func (c *Config) Zone() *time.Location {
@ -67,14 +76,26 @@ func Configure() *Config {
return c
}
// AddToBlacklist adds the given domain to the blacklist slice and persists to disk.
func (c *Config) AddToBlacklist(domain string) {
for _, d := range c.Blacklist {
if d == domain {
return
c.Blacklist = addToList(domain, c.Blacklist)
c.Save()
}
// AddToWhitelist adds the given domain to the whitelist slice and persists to disk.
func (c *Config) AddToWhitelist(domain string) {
c.Whitelist = addToList(domain, c.Whitelist)
c.Save()
}
func addToList(key string, arr []string) []string {
for _, d := range arr {
if d == key {
return arr
}
}
c.Blacklist = append(c.Blacklist, domain)
return append(arr, key)
}
func (c *Config) Save() {
@ -113,5 +134,6 @@ func defaultConfig() *Config {
UtcOffset: 60,
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
Blacklist: []string{"youtube.com"},
Whitelist: []string{"brainbaking.com"},
}
}

View File

@ -52,16 +52,48 @@ func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
}
func TestAddToBlacklistNotYetAddsToList(t *testing.T) {
func TestWhitelist(t *testing.T) {
conf := Config{
Whitelist: []string{
"youtube.com",
},
Port: 123,
Token: "token",
AllowedWebmentionSources: []string{"blah.com"},
}
t.Cleanup(func() {
os.Remove("config.json")
})
conf.AddToWhitelist("dinges.be")
assert.Contains(t, conf.Whitelist, "dinges.be")
assert.Equal(t, 2, len(conf.Whitelist))
confFromFile := Configure()
assert.Contains(t, confFromFile.Whitelist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Whitelist))
}
func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) {
conf := Config{
Blacklist: []string{
"youtube.com",
},
Port: 123,
Token: "token",
AllowedWebmentionSources: []string{"blah.com"},
}
t.Cleanup(func() {
os.Remove("config.json")
})
conf.AddToBlacklist("dinges.be")
assert.Contains(t, conf.Blacklist, "dinges.be")
assert.Equal(t, 2, len(conf.Blacklist))
confFromFile := Configure()
assert.Contains(t, confFromFile.Blacklist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Blacklist))
}
func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
@ -69,13 +101,59 @@ func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
Blacklist: []string{
"youtube.com",
},
Port: 123,
Token: "token",
AllowedWebmentionSources: []string{"blah.com"},
}
t.Cleanup(func() {
os.Remove("config.json")
})
conf.AddToBlacklist("youtube.com")
assert.Contains(t, conf.Blacklist, "youtube.com")
assert.Equal(t, 1, len(conf.Blacklist))
}
func TestIsWhitelisted(t *testing.T) {
cases := []struct {
label string
url string
expected bool
}{
{
"do not whitelist if domain is part of relative url",
"https://brainbaking.com/post/youtube.com-sucks",
false,
},
{
"whitelist if https domain is on the list",
"https://youtube.com/stuff",
true,
},
{
"whitelist if http domain is on the list",
"http://youtube.com/stuff",
true,
},
{
"do not whitelist if relative url",
"/youtube.com",
false,
},
}
conf := Config{
Whitelist: []string{
"youtube.com",
},
}
for _, tc := range cases {
t.Run(tc.label, func(t *testing.T) {
assert.Equal(t, tc.expected, conf.IsWhitelisted(tc.url))
})
}
}
func TestIsBlacklisted(t *testing.T) {
cases := []struct {
label string

View File

@ -104,14 +104,17 @@ func (r *mentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er
}
func (r *mentionRepoBunt) mentionToKey(wm mf.Mention) string {
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
return fmt.Sprintf("%s:%s", wm.Key(), wm.TargetDomain())
}
// Get returns a single unmarshalled json value based on the mention key.
// It returns the unmarshalled result or nil if something went wrong.
func (r *mentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData {
return r.getByKey(r.mentionToKey(wm))
}
func (r *mentionRepoBunt) getByKey(key string) *mf.IndiewebData {
var data mf.IndiewebData
key := r.mentionToKey(wm)
err := r.db.View(func(tx *buntdb.Tx) error {
val, err := tx.Get(key)
if err != nil {

View File

@ -46,7 +46,6 @@ func blacklistDomain(domain string) {
log.Info().Str("domain", domain).Msg("Blacklisting...")
config := common.Configure()
config.AddToBlacklist(domain)
config.Save()
repo := db.NewMentionRepo(config)
for _, domain := range config.AllowedWebmentionSources {