diff --git a/INSTALL.md b/INSTALL.md index 36f007a..b7f2d44 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -19,7 +19,6 @@ Place a `config.json` file in the same directory that looks like this: (below ar "port": 1337, "host": "localhost", "token": "miauwkes", - "conString": "mentions.db", "utcOffset": 60, "allowedWebmentionSources": [ "brainbaking.com", @@ -35,10 +34,17 @@ Place a `config.json` file in the same directory that looks like this: (below ar - token, allowedWebmentionSources: see below, used for authentication - blacklist: blacklist domains from which we do NOT send to or accept mentions from. - utcOffset: offset in minutes for date processing, starting from UTC time. -- conString: file path to store all mentions and author avatars in a simple key/value store, based on [buntdb](https://github.com/tidwall/buntdb). If the file does not exist yet, it will simply be created. If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`. +To keep things simple, the file path to store all mentions and author avatars in a simple key/value store is hardcoded and set to: + +- mentions.db (in working dir) for approved mentions +- mentions_toapprove.db (in working dir) for mentions in moderation. + +The database is based on [buntdb](https://github.com/tidwall/buntdb). If the files do not exist, they will simply be created. + + ## 3. Reverse proxy Put it behind a reverse proxy such as nginx using something like this: diff --git a/app/webmention/handler_test.go b/app/webmention/handler_test.go index b556159..bdf6515 100644 --- a/app/webmention/handler_test.go +++ b/app/webmention/handler_test.go @@ -27,7 +27,6 @@ var ( ) func init() { - cnf.ConString = ":memory:" repo = db.NewMentionRepo(cnf) } diff --git a/app/webmention/recv/receive_test.go b/app/webmention/recv/receive_test.go index 3f349a6..1f96193 100644 --- a/app/webmention/recv/receive_test.go +++ b/app/webmention/recv/receive_test.go @@ -21,7 +21,6 @@ var conf = &common.Config{ "jefklakscodex.com", "brainbaking.com", }, - ConString: ":memory:", Blacklist: []string{ "blacklisted.com", }, @@ -60,6 +59,7 @@ func TestSaveAuthorPictureLocally(t *testing.T) { }, } + t.Cleanup(db.Purge) for _, tc := range cases { t.Run(tc.label, func(t *testing.T) { repo := db.NewMentionRepo(conf) @@ -207,6 +207,7 @@ func TestReceive(t *testing.T) { func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testing.T) { repo := db.NewMentionRepo(conf) + t.Cleanup(db.Purge) wm := mf.Mention{ Source: "https://brainbaking.com", @@ -239,6 +240,7 @@ func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) { } repo := db.NewMentionRepo(conf) + t.Cleanup(db.Purge) receiver := &Receiver{ Conf: conf, Repo: repo, @@ -255,6 +257,7 @@ func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) { } repo := db.NewMentionRepo(conf) + t.Cleanup(db.Purge) receiver := &Receiver{ Conf: conf, Repo: repo, @@ -273,6 +276,7 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t * Target: "https://brainbaking.com/", } repo := db.NewMentionRepo(conf) + t.Cleanup(db.Purge) recv := &Receiver{ Conf: conf, Repo: repo, @@ -294,6 +298,7 @@ func TestProcessSourceBodyAbortsIfNoMentionOfTargetFoundInSourceHtml(t *testing. Target: "https://jefklakscodex.com/articles", } repo := db.NewMentionRepo(conf) + t.Cleanup(db.Purge) recv := &Receiver{ Conf: conf, Repo: repo, diff --git a/app/webmention/send/send_test.go b/app/webmention/send/send_test.go index b0d5175..3d55c0e 100644 --- a/app/webmention/send/send_test.go +++ b/app/webmention/send/send_test.go @@ -15,7 +15,6 @@ import ( ) var conf = &common.Config{ - ConString: ":memory:", AllowedWebmentionSources: []string{ "domain", }, @@ -135,6 +134,7 @@ func TestSendMentionIntegrationStressTest(t *testing.T) { func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) { posted := map[string]interface{}{} var lock = sync.Mutex{} + t.Cleanup(db.Purge) snder := Sender{ Conf: conf, diff --git a/common/config.go b/common/config.go index 296f85d..d9f0f13 100644 --- a/common/config.go +++ b/common/config.go @@ -16,7 +16,6 @@ type Config struct { Token string `json:"token"` UtcOffset int `json:"utcOffset"` DataPath string `json:"dataPath"` - ConString string `json:"conString"` AllowedWebmentionSources []string `json:"allowedWebmentionSources"` Blacklist []string `json:"blacklist"` } @@ -41,9 +40,6 @@ func (c *Config) missingKeys() []string { if c.Token == "" { keys = append(keys, "token") } - if c.ConString == "" { - keys = append(keys, "conString") - } if len(c.AllowedWebmentionSources) == 0 { keys = append(keys, "allowedWebmentionSources") } @@ -115,7 +111,6 @@ func defaultConfig() *Config { Port: 1337, Token: "miauwkes", UtcOffset: 60, - ConString: "mentions.db", AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"}, Blacklist: []string{"youtube.com"}, } diff --git a/common/config_test.go b/common/config_test.go index 7887f2f..fa80b03 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -8,15 +8,16 @@ import ( "testing" ) +func cleanupConfig() { + os.Remove("config.json") +} + func TestReadFromJsonMalformedReversToDefaults(t *testing.T) { - err := ioutil.WriteFile("config.json", []byte("dinges"), fs.ModePerm) - if err != nil { - assert.Failf(t, "Error writing test config.json: %s", err.Error()) - } + ioutil.WriteFile("config.json", []byte("dinges"), fs.ModePerm) + t.Cleanup(cleanupConfig) config := Configure() assert.Contains(t, config.AllowedWebmentionSources, "brainbaking.com") - os.Remove("config.json") } func TestReadFromJsonWithCorrectJsonData(t *testing.T) { @@ -24,7 +25,6 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) { "port": 1337, "host": "localhost", "token": "miauwkes", - "conString": "mentions.db", "utcOffset": 60, "allowedWebmentionSources": [ "snoopy.be" @@ -33,25 +33,23 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) { "youtube.com" ] }` - err := ioutil.WriteFile("config.json", []byte(confString), fs.ModePerm) - if err != nil { - assert.Failf(t, "Error writing test config.json: %s", err.Error()) - } + ioutil.WriteFile("config.json", []byte(confString), fs.ModePerm) + t.Cleanup(cleanupConfig) config := Configure() assert.Contains(t, config.AllowedWebmentionSources, "snoopy.be") assert.Equal(t, 1, len(config.AllowedWebmentionSources)) - os.Remove("config.json") } func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) { + t.Cleanup(cleanupConfig) + config := Configure() config.AddToBlacklist("somethingnew.be") config.Save() newConfig := Configure() assert.Contains(t, newConfig.Blacklist, "somethingnew.be") - os.Remove("config.json") } func TestAddToBlacklistNotYetAddsToList(t *testing.T) { diff --git a/db/repo.go b/db/buntrepo.go similarity index 74% rename from db/repo.go rename to db/buntrepo.go index 9cdb6b5..6593b66 100644 --- a/db/repo.go +++ b/db/buntrepo.go @@ -4,7 +4,6 @@ package db import ( "brainbaking.com/go-jamming/app/mf" - "brainbaking.com/go-jamming/common" "encoding/json" "fmt" "github.com/rs/zerolog/log" @@ -12,24 +11,12 @@ import ( "strings" ) -type MentionRepoBunt struct { +type mentionRepoBunt struct { db *buntdb.DB } -type MentionRepo interface { - Save(key mf.Mention, data *mf.IndiewebData) (string, error) - SavePicture(bytes string, domain string) (string, error) - Delete(key mf.Mention) - CleanupSpam(domain string, blacklist []string) - LastSentMention(domain string) string - UpdateLastSentMention(domain string, lastSent string) - Get(key mf.Mention) *mf.IndiewebData - GetPicture(domain string) []byte - GetAll(domain string) mf.IndiewebDataResult -} - // CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry. -func (r *MentionRepoBunt) CleanupSpam(domain string, blacklist []string) { +func (r *mentionRepoBunt) CleanupSpam(domain string, blacklist []string) { for _, mention := range r.GetAll(domain).Data { for _, blacklisted := range blacklist { if strings.Contains(mention.Url, blacklisted) { @@ -40,7 +27,7 @@ func (r *MentionRepoBunt) CleanupSpam(domain string, blacklist []string) { } // UpdateLastSentMention updates the last sent mention link. Logs but ignores errors. -func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) { +func (r *mentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) { err := r.db.Update(func(tx *buntdb.Tx) error { _, _, err := tx.Set(lastSentKey(domain), lastSentMentionLink, nil) return err @@ -51,7 +38,7 @@ func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLi } // LastSentMention fetches the last known RSS link where mentions were sent, or an empty string if an error occured. -func (r *MentionRepoBunt) LastSentMention(domain string) string { +func (r *mentionRepoBunt) LastSentMention(domain string) string { var lastSent string err := r.db.View(func(tx *buntdb.Tx) error { val, err := tx.Get(lastSentKey(domain)) @@ -70,7 +57,7 @@ func lastSentKey(domain string) string { } // Delete removes a possibly present mention by key. Ignores but logs possible errors. -func (r *MentionRepoBunt) Delete(wm mf.Mention) { +func (r *mentionRepoBunt) Delete(wm mf.Mention) { key := r.mentionToKey(wm) err := r.db.Update(func(tx *buntdb.Tx) error { _, err := tx.Delete(key) @@ -83,7 +70,7 @@ func (r *MentionRepoBunt) Delete(wm mf.Mention) { } } -func (r *MentionRepoBunt) SavePicture(bytes string, domain string) (string, error) { +func (r *mentionRepoBunt) SavePicture(bytes string, domain string) (string, error) { key := pictureKey(domain) err := r.db.Update(func(tx *buntdb.Tx) error { _, _, err := tx.Set(key, bytes, nil) @@ -100,7 +87,7 @@ func pictureKey(domain string) string { } // Save saves the mention by marshalling data. Returns the key or a marshal/persist error. -func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, error) { +func (r *mentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, error) { jsonData, err := json.Marshal(data) if err != nil { return "", err @@ -116,13 +103,13 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er return key, nil } -func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string { +func (r *mentionRepoBunt) mentionToKey(wm mf.Mention) string { return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain()) } // Get returns a single unmarshalled json value based on the mention key. // It returns the unmarshalled result or nil if something went wrong. -func (r *MentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData { +func (r *mentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData { var data mf.IndiewebData key := r.mentionToKey(wm) err := r.db.View(func(tx *buntdb.Tx) error { @@ -143,7 +130,7 @@ func (r *MentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData { return &data } -func (r *MentionRepoBunt) GetPicture(domain string) []byte { +func (r *mentionRepoBunt) GetPicture(domain string) []byte { var data []byte key := pictureKey(domain) err := r.db.View(func(tx *buntdb.Tx) error { @@ -164,7 +151,7 @@ func (r *MentionRepoBunt) GetPicture(domain string) []byte { // GetAll returns a wrapped data result for all mentions for a particular domain. // Intentionally ignores marshal errors, db should be consistent! // Warning, this will potentially marshall 10k strings! See benchmark test. -func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult { +func (r *mentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult { var data []*mf.IndiewebData err := r.db.View(func(tx *buntdb.Tx) error { return tx.Ascend(domain, func(key, value string) bool { @@ -182,20 +169,20 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult { return mf.ResultSuccess(data) } -// NewMentionRepo opens a database connection using default buntdb settings. +// NewMentionRepoBunt opens a database connection using default buntdb settings. // It also creates necessary indexes based on the passed domain config. // This panics if it cannot open the db. -func NewMentionRepo(c *common.Config) *MentionRepoBunt { - repo := &MentionRepoBunt{} - db, err := buntdb.Open(c.ConString) +func newMentionRepoBunt(conString string, allowedWebmentionSources []string) *mentionRepoBunt { + approvedRepo := &mentionRepoBunt{} + db, err := buntdb.Open(conString) if err != nil { - log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db") + log.Fatal().Str("constr", conString).Msg("new mention repo: cannot open db") } - repo.db = db + approvedRepo.db = db - for _, domain := range c.AllowedWebmentionSources { + for _, domain := range allowedWebmentionSources { db.CreateIndex(domain, fmt.Sprintf("*:%s", domain), buntdb.IndexString) } - return repo + return approvedRepo } diff --git a/db/repo_test.go b/db/buntrepo_test.go similarity index 84% rename from db/repo_test.go rename to db/buntrepo_test.go index 050d201..de13c79 100644 --- a/db/repo_test.go +++ b/db/buntrepo_test.go @@ -2,7 +2,6 @@ package db import ( "brainbaking.com/go-jamming/app/mf" - "brainbaking.com/go-jamming/common" "fmt" "github.com/stretchr/testify/assert" "io/ioutil" @@ -10,20 +9,13 @@ import ( "testing" ) -var ( - conf = &common.Config{ - ConString: ":memory:", - AllowedWebmentionSources: []string{ - "pussycat.com", - }, - } -) - func TestSaveAndGetPicture(t *testing.T) { data, err := ioutil.ReadFile("../mocks/picture.jpg") assert.NoError(t, err) - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) key, dberr := db.SavePicture(string(data), "bloeberig.be") assert.NoError(t, dberr) assert.Equal(t, "bloeberig.be:picture", key) @@ -33,7 +25,9 @@ func TestSaveAndGetPicture(t *testing.T) { } func TestCleanupSpam(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) db.Save(mf.Mention{ Source: "https://naar.hier/jup", Target: "https://pussycat.com/coolpussy.html", @@ -61,7 +55,9 @@ func TestCleanupSpam(t *testing.T) { } func TestDelete(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) wm := mf.Mention{ Target: "https://pussycat.com/coolpussy.html", } @@ -75,7 +71,9 @@ func TestDelete(t *testing.T) { } func TestUpdateLastSentMention(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) db.UpdateLastSentMention("pussycat.com", "https://last.sent") last := db.LastSentMention("pussycat.com") @@ -84,7 +82,9 @@ func TestUpdateLastSentMention(t *testing.T) { } func TestGet(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) wm := mf.Mention{ Target: "https://pussycat.com/coolpussy.html", } @@ -97,12 +97,11 @@ func TestGet(t *testing.T) { } func BenchmarkMentionRepoBunt_GetAll(b *testing.B) { - defer os.Remove("test.db") - db := NewMentionRepo(&common.Config{ - ConString: "test.db", - AllowedWebmentionSources: []string{ - "pussycat.com", - }, + b.Cleanup(func() { + os.Remove("test.db") + }) + db := newMentionRepoBunt("test.db", []string{ + "pussycat.com", }) items := 10000 @@ -127,7 +126,9 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) { } func TestGetAllAndSaveSomeJson(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) db.Save(mf.Mention{ Target: "https://pussycat.com/coolpussy.html", }, &mf.IndiewebData{ @@ -140,7 +141,9 @@ func TestGetAllAndSaveSomeJson(t *testing.T) { } func TestGetFiltersBasedOnDomain(t *testing.T) { - db := NewMentionRepo(conf) + db := newMentionRepoBunt(":memory:", []string{ + "pussycat.com", + }) db.Save(mf.Mention{ Target: "https://pussycat.com/coolpussy.html", }, &mf.IndiewebData{ diff --git a/db/mentionrepo.go b/db/mentionrepo.go new file mode 100644 index 0000000..5d420ae --- /dev/null +++ b/db/mentionrepo.go @@ -0,0 +1,101 @@ +package db + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "os" +) + +type MentionRepo interface { + InModeration(key mf.Mention, data *mf.IndiewebData) (string, error) + Save(key mf.Mention, data *mf.IndiewebData) (string, error) + Delete(key mf.Mention) + Approve(key mf.Mention) + Reject(key mf.Mention) + + Get(key mf.Mention) *mf.IndiewebData + GetAll(domain string) mf.IndiewebDataResult + GetAllToModerate(domain string) mf.IndiewebDataResult + + CleanupSpam(domain string, blacklist []string) + + SavePicture(bytes string, domain string) (string, error) + GetPicture(domain string) []byte + LastSentMention(domain string) string + UpdateLastSentMention(domain string, lastSent string) +} + +type MentionRepoWrapper struct { + toApproveRepo *mentionRepoBunt + approvedRepo *mentionRepoBunt +} + +func (m MentionRepoWrapper) Save(key mf.Mention, data *mf.IndiewebData) (string, error) { + return m.approvedRepo.Save(key, data) +} + +func (m MentionRepoWrapper) InModeration(key mf.Mention, data *mf.IndiewebData) (string, error) { + return m.toApproveRepo.Save(key, data) +} + +func (m MentionRepoWrapper) SavePicture(bytes string, domain string) (string, error) { + return m.approvedRepo.SavePicture(bytes, domain) +} + +func (m MentionRepoWrapper) Delete(key mf.Mention) { + m.approvedRepo.Delete(key) +} + +func (m MentionRepoWrapper) Approve(keyInModeration mf.Mention) { + toApprove := m.toApproveRepo.Get(keyInModeration) + m.Save(keyInModeration, toApprove) + m.toApproveRepo.Delete(keyInModeration) +} + +func (m MentionRepoWrapper) Reject(keyInModeration mf.Mention) { + m.toApproveRepo.Delete(keyInModeration) +} + +func (m MentionRepoWrapper) CleanupSpam(domain string, blacklist []string) { + m.approvedRepo.CleanupSpam(domain, blacklist) +} + +func (m MentionRepoWrapper) LastSentMention(domain string) string { + return m.approvedRepo.LastSentMention(domain) +} + +func (m MentionRepoWrapper) UpdateLastSentMention(domain string, lastSent string) { + m.approvedRepo.UpdateLastSentMention(domain, lastSent) +} + +func (m MentionRepoWrapper) Get(key mf.Mention) *mf.IndiewebData { + return m.approvedRepo.Get(key) +} + +func (m MentionRepoWrapper) GetPicture(domain string) []byte { + return m.approvedRepo.GetPicture(domain) +} + +func (m MentionRepoWrapper) GetAll(domain string) mf.IndiewebDataResult { + return m.approvedRepo.GetAll(domain) +} + +func (m MentionRepoWrapper) GetAllToModerate(domain string) mf.IndiewebDataResult { + return m.toApproveRepo.GetAll(domain) +} + +// NewMentionRepo returns a wrapper to two different mentionRepoBunt instances +// Depending on the to approve or approved mention, it will be saved in another file. +func NewMentionRepo(c *common.Config) *MentionRepoWrapper { + return &MentionRepoWrapper{ + toApproveRepo: newMentionRepoBunt("mentions_toapprove.db", c.AllowedWebmentionSources), + approvedRepo: newMentionRepoBunt("mentions.db", c.AllowedWebmentionSources), + } +} + +// Purge removes all database files from disk. +// This is dangerous in production and should be used as a shorthand in tests! +func Purge() { + os.Remove("mentions_toapprove.db") + os.Remove("mentions.db") +} diff --git a/db/mentionrepo_test.go b/db/mentionrepo_test.go new file mode 100644 index 0000000..ce2944c --- /dev/null +++ b/db/mentionrepo_test.go @@ -0,0 +1,64 @@ +package db + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "github.com/stretchr/testify/assert" + "testing" +) + +var ( + repoCnf = &common.Config{ + AllowedWebmentionSources: []string{ + "brainbaking.com", + }, + } +) + +func TestApproveCases(t *testing.T) { + cases := []struct { + label string + approve bool + expectedInModerationDb int + expectedInMentionDb int + }{ + { + "approve moves from the to moderate db to the mention db", + true, + 0, + 1, + }, + { + "reject deletes from to moderate db and leaves mention db alone", + false, + 0, + 0, + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + repo := NewMentionRepo(repoCnf) + defer Purge() + + wm := mf.Mention{ + Target: "https://brainbaking.com/sjiekedinges.html", + } + data := &mf.IndiewebData{ + Name: "lolz", + } + repo.InModeration(wm, data) + + if tc.approve { + repo.Approve(wm) + } else { + repo.Reject(wm) + } + + allWms := repo.GetAll("brainbaking.com") + allWmsToModerate := repo.GetAllToModerate("brainbaking.com") + assert.Equal(t, tc.expectedInMentionDb, len(allWms.Data), "mention db expectation failed") + assert.Equal(t, tc.expectedInModerationDb, len(allWmsToModerate.Data), "in moderation db expectation failed") + }) + } +} diff --git a/db/migrate.go b/db/migrate.go index c68e0df..ac6d667 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -10,5 +10,6 @@ func Migrate() { repo := NewMentionRepo(cnf) // no migrations needed anymore/yet - repo.db.Shrink() + repo.approvedRepo.db.Shrink() + repo.toApproveRepo.db.Shrink() }