forked from wgroeneveld/go-jamming
refactor out conString; create a db wrapper for mentions in moderation
This commit is contained in:
parent
9d08d35576
commit
3ec6694757
10
INSTALL.md
10
INSTALL.md
|
@ -19,7 +19,6 @@ Place a `config.json` file in the same directory that looks like this: (below ar
|
||||||
"port": 1337,
|
"port": 1337,
|
||||||
"host": "localhost",
|
"host": "localhost",
|
||||||
"token": "miauwkes",
|
"token": "miauwkes",
|
||||||
"conString": "mentions.db",
|
|
||||||
"utcOffset": 60,
|
"utcOffset": 60,
|
||||||
"allowedWebmentionSources": [
|
"allowedWebmentionSources": [
|
||||||
"brainbaking.com",
|
"brainbaking.com",
|
||||||
|
@ -35,10 +34,17 @@ Place a `config.json` file in the same directory that looks like this: (below ar
|
||||||
- token, allowedWebmentionSources: see below, used for authentication
|
- token, allowedWebmentionSources: see below, used for authentication
|
||||||
- blacklist: blacklist domains from which we do NOT send to or accept mentions from.
|
- blacklist: blacklist domains from which we do NOT send to or accept mentions from.
|
||||||
- utcOffset: offset in minutes for date processing, starting from UTC time.
|
- utcOffset: offset in minutes for date processing, starting from UTC time.
|
||||||
- conString: file path to store all mentions and author avatars in a simple key/value store, based on [buntdb](https://github.com/tidwall/buntdb). If the file does not exist yet, it will simply be created.
|
|
||||||
|
|
||||||
If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`.
|
If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`.
|
||||||
|
|
||||||
|
To keep things simple, the file path to store all mentions and author avatars in a simple key/value store is hardcoded and set to:
|
||||||
|
|
||||||
|
- mentions.db (in working dir) for approved mentions
|
||||||
|
- mentions_toapprove.db (in working dir) for mentions in moderation.
|
||||||
|
|
||||||
|
The database is based on [buntdb](https://github.com/tidwall/buntdb). If the files do not exist, they will simply be created.
|
||||||
|
|
||||||
|
|
||||||
## 3. Reverse proxy
|
## 3. Reverse proxy
|
||||||
|
|
||||||
Put it behind a reverse proxy such as nginx using something like this:
|
Put it behind a reverse proxy such as nginx using something like this:
|
||||||
|
|
|
@ -27,7 +27,6 @@ var (
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
cnf.ConString = ":memory:"
|
|
||||||
repo = db.NewMentionRepo(cnf)
|
repo = db.NewMentionRepo(cnf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -21,7 +21,6 @@ var conf = &common.Config{
|
||||||
"jefklakscodex.com",
|
"jefklakscodex.com",
|
||||||
"brainbaking.com",
|
"brainbaking.com",
|
||||||
},
|
},
|
||||||
ConString: ":memory:",
|
|
||||||
Blacklist: []string{
|
Blacklist: []string{
|
||||||
"blacklisted.com",
|
"blacklisted.com",
|
||||||
},
|
},
|
||||||
|
@ -60,6 +59,7 @@ func TestSaveAuthorPictureLocally(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.label, func(t *testing.T) {
|
t.Run(tc.label, func(t *testing.T) {
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
@ -207,6 +207,7 @@ func TestReceive(t *testing.T) {
|
||||||
|
|
||||||
func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testing.T) {
|
func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testing.T) {
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
|
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
Source: "https://brainbaking.com",
|
Source: "https://brainbaking.com",
|
||||||
|
@ -239,6 +240,7 @@ func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
receiver := &Receiver{
|
receiver := &Receiver{
|
||||||
Conf: conf,
|
Conf: conf,
|
||||||
Repo: repo,
|
Repo: repo,
|
||||||
|
@ -255,6 +257,7 @@ func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
receiver := &Receiver{
|
receiver := &Receiver{
|
||||||
Conf: conf,
|
Conf: conf,
|
||||||
Repo: repo,
|
Repo: repo,
|
||||||
|
@ -273,6 +276,7 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *
|
||||||
Target: "https://brainbaking.com/",
|
Target: "https://brainbaking.com/",
|
||||||
}
|
}
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
recv := &Receiver{
|
recv := &Receiver{
|
||||||
Conf: conf,
|
Conf: conf,
|
||||||
Repo: repo,
|
Repo: repo,
|
||||||
|
@ -294,6 +298,7 @@ func TestProcessSourceBodyAbortsIfNoMentionOfTargetFoundInSourceHtml(t *testing.
|
||||||
Target: "https://jefklakscodex.com/articles",
|
Target: "https://jefklakscodex.com/articles",
|
||||||
}
|
}
|
||||||
repo := db.NewMentionRepo(conf)
|
repo := db.NewMentionRepo(conf)
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
recv := &Receiver{
|
recv := &Receiver{
|
||||||
Conf: conf,
|
Conf: conf,
|
||||||
Repo: repo,
|
Repo: repo,
|
||||||
|
|
|
@ -15,7 +15,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var conf = &common.Config{
|
var conf = &common.Config{
|
||||||
ConString: ":memory:",
|
|
||||||
AllowedWebmentionSources: []string{
|
AllowedWebmentionSources: []string{
|
||||||
"domain",
|
"domain",
|
||||||
},
|
},
|
||||||
|
@ -135,6 +134,7 @@ func TestSendMentionIntegrationStressTest(t *testing.T) {
|
||||||
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
|
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
|
||||||
posted := map[string]interface{}{}
|
posted := map[string]interface{}{}
|
||||||
var lock = sync.Mutex{}
|
var lock = sync.Mutex{}
|
||||||
|
t.Cleanup(db.Purge)
|
||||||
|
|
||||||
snder := Sender{
|
snder := Sender{
|
||||||
Conf: conf,
|
Conf: conf,
|
||||||
|
|
|
@ -16,7 +16,6 @@ type Config struct {
|
||||||
Token string `json:"token"`
|
Token string `json:"token"`
|
||||||
UtcOffset int `json:"utcOffset"`
|
UtcOffset int `json:"utcOffset"`
|
||||||
DataPath string `json:"dataPath"`
|
DataPath string `json:"dataPath"`
|
||||||
ConString string `json:"conString"`
|
|
||||||
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
||||||
Blacklist []string `json:"blacklist"`
|
Blacklist []string `json:"blacklist"`
|
||||||
}
|
}
|
||||||
|
@ -41,9 +40,6 @@ func (c *Config) missingKeys() []string {
|
||||||
if c.Token == "" {
|
if c.Token == "" {
|
||||||
keys = append(keys, "token")
|
keys = append(keys, "token")
|
||||||
}
|
}
|
||||||
if c.ConString == "" {
|
|
||||||
keys = append(keys, "conString")
|
|
||||||
}
|
|
||||||
if len(c.AllowedWebmentionSources) == 0 {
|
if len(c.AllowedWebmentionSources) == 0 {
|
||||||
keys = append(keys, "allowedWebmentionSources")
|
keys = append(keys, "allowedWebmentionSources")
|
||||||
}
|
}
|
||||||
|
@ -115,7 +111,6 @@ func defaultConfig() *Config {
|
||||||
Port: 1337,
|
Port: 1337,
|
||||||
Token: "miauwkes",
|
Token: "miauwkes",
|
||||||
UtcOffset: 60,
|
UtcOffset: 60,
|
||||||
ConString: "mentions.db",
|
|
||||||
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
||||||
Blacklist: []string{"youtube.com"},
|
Blacklist: []string{"youtube.com"},
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,15 +8,16 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func cleanupConfig() {
|
||||||
|
os.Remove("config.json")
|
||||||
|
}
|
||||||
|
|
||||||
func TestReadFromJsonMalformedReversToDefaults(t *testing.T) {
|
func TestReadFromJsonMalformedReversToDefaults(t *testing.T) {
|
||||||
err := ioutil.WriteFile("config.json", []byte("dinges"), fs.ModePerm)
|
ioutil.WriteFile("config.json", []byte("dinges"), fs.ModePerm)
|
||||||
if err != nil {
|
t.Cleanup(cleanupConfig)
|
||||||
assert.Failf(t, "Error writing test config.json: %s", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
config := Configure()
|
config := Configure()
|
||||||
assert.Contains(t, config.AllowedWebmentionSources, "brainbaking.com")
|
assert.Contains(t, config.AllowedWebmentionSources, "brainbaking.com")
|
||||||
os.Remove("config.json")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
||||||
|
@ -24,7 +25,6 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
||||||
"port": 1337,
|
"port": 1337,
|
||||||
"host": "localhost",
|
"host": "localhost",
|
||||||
"token": "miauwkes",
|
"token": "miauwkes",
|
||||||
"conString": "mentions.db",
|
|
||||||
"utcOffset": 60,
|
"utcOffset": 60,
|
||||||
"allowedWebmentionSources": [
|
"allowedWebmentionSources": [
|
||||||
"snoopy.be"
|
"snoopy.be"
|
||||||
|
@ -33,25 +33,23 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
||||||
"youtube.com"
|
"youtube.com"
|
||||||
]
|
]
|
||||||
}`
|
}`
|
||||||
err := ioutil.WriteFile("config.json", []byte(confString), fs.ModePerm)
|
ioutil.WriteFile("config.json", []byte(confString), fs.ModePerm)
|
||||||
if err != nil {
|
t.Cleanup(cleanupConfig)
|
||||||
assert.Failf(t, "Error writing test config.json: %s", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
config := Configure()
|
config := Configure()
|
||||||
assert.Contains(t, config.AllowedWebmentionSources, "snoopy.be")
|
assert.Contains(t, config.AllowedWebmentionSources, "snoopy.be")
|
||||||
assert.Equal(t, 1, len(config.AllowedWebmentionSources))
|
assert.Equal(t, 1, len(config.AllowedWebmentionSources))
|
||||||
os.Remove("config.json")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
|
func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
|
||||||
|
t.Cleanup(cleanupConfig)
|
||||||
|
|
||||||
config := Configure()
|
config := Configure()
|
||||||
config.AddToBlacklist("somethingnew.be")
|
config.AddToBlacklist("somethingnew.be")
|
||||||
config.Save()
|
config.Save()
|
||||||
|
|
||||||
newConfig := Configure()
|
newConfig := Configure()
|
||||||
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
|
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
|
||||||
os.Remove("config.json")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAddToBlacklistNotYetAddsToList(t *testing.T) {
|
func TestAddToBlacklistNotYetAddsToList(t *testing.T) {
|
||||||
|
|
|
@ -4,7 +4,6 @@ package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"brainbaking.com/go-jamming/app/mf"
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
"brainbaking.com/go-jamming/common"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
|
@ -12,24 +11,12 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
type MentionRepoBunt struct {
|
type mentionRepoBunt struct {
|
||||||
db *buntdb.DB
|
db *buntdb.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
type MentionRepo interface {
|
|
||||||
Save(key mf.Mention, data *mf.IndiewebData) (string, error)
|
|
||||||
SavePicture(bytes string, domain string) (string, error)
|
|
||||||
Delete(key mf.Mention)
|
|
||||||
CleanupSpam(domain string, blacklist []string)
|
|
||||||
LastSentMention(domain string) string
|
|
||||||
UpdateLastSentMention(domain string, lastSent string)
|
|
||||||
Get(key mf.Mention) *mf.IndiewebData
|
|
||||||
GetPicture(domain string) []byte
|
|
||||||
GetAll(domain string) mf.IndiewebDataResult
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry.
|
// CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry.
|
||||||
func (r *MentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
|
func (r *mentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
|
||||||
for _, mention := range r.GetAll(domain).Data {
|
for _, mention := range r.GetAll(domain).Data {
|
||||||
for _, blacklisted := range blacklist {
|
for _, blacklisted := range blacklist {
|
||||||
if strings.Contains(mention.Url, blacklisted) {
|
if strings.Contains(mention.Url, blacklisted) {
|
||||||
|
@ -40,7 +27,7 @@ func (r *MentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateLastSentMention updates the last sent mention link. Logs but ignores errors.
|
// UpdateLastSentMention updates the last sent mention link. Logs but ignores errors.
|
||||||
func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) {
|
func (r *mentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) {
|
||||||
err := r.db.Update(func(tx *buntdb.Tx) error {
|
err := r.db.Update(func(tx *buntdb.Tx) error {
|
||||||
_, _, err := tx.Set(lastSentKey(domain), lastSentMentionLink, nil)
|
_, _, err := tx.Set(lastSentKey(domain), lastSentMentionLink, nil)
|
||||||
return err
|
return err
|
||||||
|
@ -51,7 +38,7 @@ func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLi
|
||||||
}
|
}
|
||||||
|
|
||||||
// LastSentMention fetches the last known RSS link where mentions were sent, or an empty string if an error occured.
|
// LastSentMention fetches the last known RSS link where mentions were sent, or an empty string if an error occured.
|
||||||
func (r *MentionRepoBunt) LastSentMention(domain string) string {
|
func (r *mentionRepoBunt) LastSentMention(domain string) string {
|
||||||
var lastSent string
|
var lastSent string
|
||||||
err := r.db.View(func(tx *buntdb.Tx) error {
|
err := r.db.View(func(tx *buntdb.Tx) error {
|
||||||
val, err := tx.Get(lastSentKey(domain))
|
val, err := tx.Get(lastSentKey(domain))
|
||||||
|
@ -70,7 +57,7 @@ func lastSentKey(domain string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete removes a possibly present mention by key. Ignores but logs possible errors.
|
// Delete removes a possibly present mention by key. Ignores but logs possible errors.
|
||||||
func (r *MentionRepoBunt) Delete(wm mf.Mention) {
|
func (r *mentionRepoBunt) Delete(wm mf.Mention) {
|
||||||
key := r.mentionToKey(wm)
|
key := r.mentionToKey(wm)
|
||||||
err := r.db.Update(func(tx *buntdb.Tx) error {
|
err := r.db.Update(func(tx *buntdb.Tx) error {
|
||||||
_, err := tx.Delete(key)
|
_, err := tx.Delete(key)
|
||||||
|
@ -83,7 +70,7 @@ func (r *MentionRepoBunt) Delete(wm mf.Mention) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *MentionRepoBunt) SavePicture(bytes string, domain string) (string, error) {
|
func (r *mentionRepoBunt) SavePicture(bytes string, domain string) (string, error) {
|
||||||
key := pictureKey(domain)
|
key := pictureKey(domain)
|
||||||
err := r.db.Update(func(tx *buntdb.Tx) error {
|
err := r.db.Update(func(tx *buntdb.Tx) error {
|
||||||
_, _, err := tx.Set(key, bytes, nil)
|
_, _, err := tx.Set(key, bytes, nil)
|
||||||
|
@ -100,7 +87,7 @@ func pictureKey(domain string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save saves the mention by marshalling data. Returns the key or a marshal/persist error.
|
// Save saves the mention by marshalling data. Returns the key or a marshal/persist error.
|
||||||
func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, error) {
|
func (r *mentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, error) {
|
||||||
jsonData, err := json.Marshal(data)
|
jsonData, err := json.Marshal(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
@ -116,13 +103,13 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er
|
||||||
return key, nil
|
return key, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string {
|
func (r *mentionRepoBunt) mentionToKey(wm mf.Mention) string {
|
||||||
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
|
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns a single unmarshalled json value based on the mention key.
|
// Get returns a single unmarshalled json value based on the mention key.
|
||||||
// It returns the unmarshalled result or nil if something went wrong.
|
// It returns the unmarshalled result or nil if something went wrong.
|
||||||
func (r *MentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData {
|
func (r *mentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData {
|
||||||
var data mf.IndiewebData
|
var data mf.IndiewebData
|
||||||
key := r.mentionToKey(wm)
|
key := r.mentionToKey(wm)
|
||||||
err := r.db.View(func(tx *buntdb.Tx) error {
|
err := r.db.View(func(tx *buntdb.Tx) error {
|
||||||
|
@ -143,7 +130,7 @@ func (r *MentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData {
|
||||||
return &data
|
return &data
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *MentionRepoBunt) GetPicture(domain string) []byte {
|
func (r *mentionRepoBunt) GetPicture(domain string) []byte {
|
||||||
var data []byte
|
var data []byte
|
||||||
key := pictureKey(domain)
|
key := pictureKey(domain)
|
||||||
err := r.db.View(func(tx *buntdb.Tx) error {
|
err := r.db.View(func(tx *buntdb.Tx) error {
|
||||||
|
@ -164,7 +151,7 @@ func (r *MentionRepoBunt) GetPicture(domain string) []byte {
|
||||||
// GetAll returns a wrapped data result for all mentions for a particular domain.
|
// GetAll returns a wrapped data result for all mentions for a particular domain.
|
||||||
// Intentionally ignores marshal errors, db should be consistent!
|
// Intentionally ignores marshal errors, db should be consistent!
|
||||||
// Warning, this will potentially marshall 10k strings! See benchmark test.
|
// Warning, this will potentially marshall 10k strings! See benchmark test.
|
||||||
func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
|
func (r *mentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
|
||||||
var data []*mf.IndiewebData
|
var data []*mf.IndiewebData
|
||||||
err := r.db.View(func(tx *buntdb.Tx) error {
|
err := r.db.View(func(tx *buntdb.Tx) error {
|
||||||
return tx.Ascend(domain, func(key, value string) bool {
|
return tx.Ascend(domain, func(key, value string) bool {
|
||||||
|
@ -182,20 +169,20 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
|
||||||
return mf.ResultSuccess(data)
|
return mf.ResultSuccess(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMentionRepo opens a database connection using default buntdb settings.
|
// NewMentionRepoBunt opens a database connection using default buntdb settings.
|
||||||
// It also creates necessary indexes based on the passed domain config.
|
// It also creates necessary indexes based on the passed domain config.
|
||||||
// This panics if it cannot open the db.
|
// This panics if it cannot open the db.
|
||||||
func NewMentionRepo(c *common.Config) *MentionRepoBunt {
|
func newMentionRepoBunt(conString string, allowedWebmentionSources []string) *mentionRepoBunt {
|
||||||
repo := &MentionRepoBunt{}
|
approvedRepo := &mentionRepoBunt{}
|
||||||
db, err := buntdb.Open(c.ConString)
|
db, err := buntdb.Open(conString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db")
|
log.Fatal().Str("constr", conString).Msg("new mention repo: cannot open db")
|
||||||
}
|
}
|
||||||
repo.db = db
|
approvedRepo.db = db
|
||||||
|
|
||||||
for _, domain := range c.AllowedWebmentionSources {
|
for _, domain := range allowedWebmentionSources {
|
||||||
db.CreateIndex(domain, fmt.Sprintf("*:%s", domain), buntdb.IndexString)
|
db.CreateIndex(domain, fmt.Sprintf("*:%s", domain), buntdb.IndexString)
|
||||||
}
|
}
|
||||||
|
|
||||||
return repo
|
return approvedRepo
|
||||||
}
|
}
|
|
@ -2,7 +2,6 @@ package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"brainbaking.com/go-jamming/app/mf"
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
"brainbaking.com/go-jamming/common"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
@ -10,20 +9,13 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
conf = &common.Config{
|
|
||||||
ConString: ":memory:",
|
|
||||||
AllowedWebmentionSources: []string{
|
|
||||||
"pussycat.com",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSaveAndGetPicture(t *testing.T) {
|
func TestSaveAndGetPicture(t *testing.T) {
|
||||||
data, err := ioutil.ReadFile("../mocks/picture.jpg")
|
data, err := ioutil.ReadFile("../mocks/picture.jpg")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
key, dberr := db.SavePicture(string(data), "bloeberig.be")
|
key, dberr := db.SavePicture(string(data), "bloeberig.be")
|
||||||
assert.NoError(t, dberr)
|
assert.NoError(t, dberr)
|
||||||
assert.Equal(t, "bloeberig.be:picture", key)
|
assert.Equal(t, "bloeberig.be:picture", key)
|
||||||
|
@ -33,7 +25,9 @@ func TestSaveAndGetPicture(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCleanupSpam(t *testing.T) {
|
func TestCleanupSpam(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Source: "https://naar.hier/jup",
|
Source: "https://naar.hier/jup",
|
||||||
Target: "https://pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
|
@ -61,7 +55,9 @@ func TestCleanupSpam(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDelete(t *testing.T) {
|
func TestDelete(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
Target: "https://pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}
|
}
|
||||||
|
@ -75,7 +71,9 @@ func TestDelete(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUpdateLastSentMention(t *testing.T) {
|
func TestUpdateLastSentMention(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
|
|
||||||
db.UpdateLastSentMention("pussycat.com", "https://last.sent")
|
db.UpdateLastSentMention("pussycat.com", "https://last.sent")
|
||||||
last := db.LastSentMention("pussycat.com")
|
last := db.LastSentMention("pussycat.com")
|
||||||
|
@ -84,7 +82,9 @@ func TestUpdateLastSentMention(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGet(t *testing.T) {
|
func TestGet(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
Target: "https://pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}
|
}
|
||||||
|
@ -97,12 +97,11 @@ func TestGet(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
||||||
defer os.Remove("test.db")
|
b.Cleanup(func() {
|
||||||
db := NewMentionRepo(&common.Config{
|
os.Remove("test.db")
|
||||||
ConString: "test.db",
|
})
|
||||||
AllowedWebmentionSources: []string{
|
db := newMentionRepoBunt("test.db", []string{
|
||||||
"pussycat.com",
|
"pussycat.com",
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
items := 10000
|
items := 10000
|
||||||
|
@ -127,7 +126,9 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetAllAndSaveSomeJson(t *testing.T) {
|
func TestGetAllAndSaveSomeJson(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Target: "https://pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}, &mf.IndiewebData{
|
}, &mf.IndiewebData{
|
||||||
|
@ -140,7 +141,9 @@ func TestGetAllAndSaveSomeJson(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetFiltersBasedOnDomain(t *testing.T) {
|
func TestGetFiltersBasedOnDomain(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := newMentionRepoBunt(":memory:", []string{
|
||||||
|
"pussycat.com",
|
||||||
|
})
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Target: "https://pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}, &mf.IndiewebData{
|
}, &mf.IndiewebData{
|
|
@ -0,0 +1,101 @@
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
|
"brainbaking.com/go-jamming/common"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MentionRepo interface {
|
||||||
|
InModeration(key mf.Mention, data *mf.IndiewebData) (string, error)
|
||||||
|
Save(key mf.Mention, data *mf.IndiewebData) (string, error)
|
||||||
|
Delete(key mf.Mention)
|
||||||
|
Approve(key mf.Mention)
|
||||||
|
Reject(key mf.Mention)
|
||||||
|
|
||||||
|
Get(key mf.Mention) *mf.IndiewebData
|
||||||
|
GetAll(domain string) mf.IndiewebDataResult
|
||||||
|
GetAllToModerate(domain string) mf.IndiewebDataResult
|
||||||
|
|
||||||
|
CleanupSpam(domain string, blacklist []string)
|
||||||
|
|
||||||
|
SavePicture(bytes string, domain string) (string, error)
|
||||||
|
GetPicture(domain string) []byte
|
||||||
|
LastSentMention(domain string) string
|
||||||
|
UpdateLastSentMention(domain string, lastSent string)
|
||||||
|
}
|
||||||
|
|
||||||
|
type MentionRepoWrapper struct {
|
||||||
|
toApproveRepo *mentionRepoBunt
|
||||||
|
approvedRepo *mentionRepoBunt
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) Save(key mf.Mention, data *mf.IndiewebData) (string, error) {
|
||||||
|
return m.approvedRepo.Save(key, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) InModeration(key mf.Mention, data *mf.IndiewebData) (string, error) {
|
||||||
|
return m.toApproveRepo.Save(key, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) SavePicture(bytes string, domain string) (string, error) {
|
||||||
|
return m.approvedRepo.SavePicture(bytes, domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) Delete(key mf.Mention) {
|
||||||
|
m.approvedRepo.Delete(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) Approve(keyInModeration mf.Mention) {
|
||||||
|
toApprove := m.toApproveRepo.Get(keyInModeration)
|
||||||
|
m.Save(keyInModeration, toApprove)
|
||||||
|
m.toApproveRepo.Delete(keyInModeration)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) Reject(keyInModeration mf.Mention) {
|
||||||
|
m.toApproveRepo.Delete(keyInModeration)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) CleanupSpam(domain string, blacklist []string) {
|
||||||
|
m.approvedRepo.CleanupSpam(domain, blacklist)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) LastSentMention(domain string) string {
|
||||||
|
return m.approvedRepo.LastSentMention(domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) UpdateLastSentMention(domain string, lastSent string) {
|
||||||
|
m.approvedRepo.UpdateLastSentMention(domain, lastSent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) Get(key mf.Mention) *mf.IndiewebData {
|
||||||
|
return m.approvedRepo.Get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) GetPicture(domain string) []byte {
|
||||||
|
return m.approvedRepo.GetPicture(domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) GetAll(domain string) mf.IndiewebDataResult {
|
||||||
|
return m.approvedRepo.GetAll(domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MentionRepoWrapper) GetAllToModerate(domain string) mf.IndiewebDataResult {
|
||||||
|
return m.toApproveRepo.GetAll(domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMentionRepo returns a wrapper to two different mentionRepoBunt instances
|
||||||
|
// Depending on the to approve or approved mention, it will be saved in another file.
|
||||||
|
func NewMentionRepo(c *common.Config) *MentionRepoWrapper {
|
||||||
|
return &MentionRepoWrapper{
|
||||||
|
toApproveRepo: newMentionRepoBunt("mentions_toapprove.db", c.AllowedWebmentionSources),
|
||||||
|
approvedRepo: newMentionRepoBunt("mentions.db", c.AllowedWebmentionSources),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Purge removes all database files from disk.
|
||||||
|
// This is dangerous in production and should be used as a shorthand in tests!
|
||||||
|
func Purge() {
|
||||||
|
os.Remove("mentions_toapprove.db")
|
||||||
|
os.Remove("mentions.db")
|
||||||
|
}
|
|
@ -0,0 +1,64 @@
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
|
"brainbaking.com/go-jamming/common"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
repoCnf = &common.Config{
|
||||||
|
AllowedWebmentionSources: []string{
|
||||||
|
"brainbaking.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApproveCases(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
label string
|
||||||
|
approve bool
|
||||||
|
expectedInModerationDb int
|
||||||
|
expectedInMentionDb int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"approve moves from the to moderate db to the mention db",
|
||||||
|
true,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"reject deletes from to moderate db and leaves mention db alone",
|
||||||
|
false,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.label, func(t *testing.T) {
|
||||||
|
repo := NewMentionRepo(repoCnf)
|
||||||
|
defer Purge()
|
||||||
|
|
||||||
|
wm := mf.Mention{
|
||||||
|
Target: "https://brainbaking.com/sjiekedinges.html",
|
||||||
|
}
|
||||||
|
data := &mf.IndiewebData{
|
||||||
|
Name: "lolz",
|
||||||
|
}
|
||||||
|
repo.InModeration(wm, data)
|
||||||
|
|
||||||
|
if tc.approve {
|
||||||
|
repo.Approve(wm)
|
||||||
|
} else {
|
||||||
|
repo.Reject(wm)
|
||||||
|
}
|
||||||
|
|
||||||
|
allWms := repo.GetAll("brainbaking.com")
|
||||||
|
allWmsToModerate := repo.GetAllToModerate("brainbaking.com")
|
||||||
|
assert.Equal(t, tc.expectedInMentionDb, len(allWms.Data), "mention db expectation failed")
|
||||||
|
assert.Equal(t, tc.expectedInModerationDb, len(allWmsToModerate.Data), "in moderation db expectation failed")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -10,5 +10,6 @@ func Migrate() {
|
||||||
repo := NewMentionRepo(cnf)
|
repo := NewMentionRepo(cnf)
|
||||||
|
|
||||||
// no migrations needed anymore/yet
|
// no migrations needed anymore/yet
|
||||||
repo.db.Shrink()
|
repo.approvedRepo.db.Shrink()
|
||||||
|
repo.toApproveRepo.db.Shrink()
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue