another attempt at fighting spam
This commit is contained in:
parent
2b3e3f9b22
commit
d0eddc3047
15
README.md
15
README.md
|
@ -181,3 +181,18 @@ That's pretty flexible. I have not taken the trouble to put this into the config
|
||||||
A separate goroutine cleans up ips each 2 minutes, the TTL is 5 minutes. See `limiter.go`.
|
A separate goroutine cleans up ips each 2 minutes, the TTL is 5 minutes. See `limiter.go`.
|
||||||
|
|
||||||
Database migrations are run using the `-migrate` flag.
|
Database migrations are run using the `-migrate` flag.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fighting spam
|
||||||
|
|
||||||
|
Since Go-jamming still supports Pingbacks, spam could be an issue. However, if the URL doesn't contain a genuine link, the mention will be immediately dropped.
|
||||||
|
|
||||||
|
Still, spammers always find a way and sometimes even create fake blog posts with real links to your blog. In that case, simply add the domain to the `blacklist` in `config.json`.
|
||||||
|
|
||||||
|
Adding this **manually** will not remove existing spam in your DB! The `-blacklist` flag is there to:
|
||||||
|
|
||||||
|
1. Automatically add it to the `blacklist` array in the config file;
|
||||||
|
2. Automatically search the DB for all allowed domains for spam from the blacklist and remove it. (Check for string match on the URL)
|
||||||
|
|
||||||
|
How to use: `./go-jamming -blacklist annoyingspam.com`. This will exit after the above actions. Then you can simply restart the server with `./go-jamming`.
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
|
"io/fs"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -70,6 +71,24 @@ func Configure() *Config {
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) AddToBlacklist(domain string) {
|
||||||
|
for _, d := range c.Blacklist {
|
||||||
|
if d == domain {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Blacklist = append(c.Blacklist, domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) Save() {
|
||||||
|
bytes, _ := json.Marshal(c) // we assume a correct internral state here
|
||||||
|
err := ioutil.WriteFile("config.json", bytes, fs.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
log.Err(err).Msg("Unable to save config.json to disk!")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func config() *Config {
|
func config() *Config {
|
||||||
confData, err := ioutil.ReadFile("config.json")
|
confData, err := ioutil.ReadFile("config.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -2,9 +2,82 @@ package common
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"io/fs"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestReadFromJsonMalformedReversToDefaults(t *testing.T) {
|
||||||
|
err := ioutil.WriteFile("config.json", []byte("dinges"), fs.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
assert.Failf(t, "Error writing test config.json: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
config := Configure()
|
||||||
|
assert.Contains(t, config.AllowedWebmentionSources, "brainbaking.com")
|
||||||
|
os.Remove("config.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
||||||
|
confString := `{
|
||||||
|
"port": 1337,
|
||||||
|
"host": "localhost",
|
||||||
|
"token": "miauwkes",
|
||||||
|
"conString": "mentions.db",
|
||||||
|
"utcOffset": 60,
|
||||||
|
"allowedWebmentionSources": [
|
||||||
|
"snoopy.be"
|
||||||
|
],
|
||||||
|
"blacklist": [
|
||||||
|
"youtube.com"
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
err := ioutil.WriteFile("config.json", []byte(confString), fs.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
assert.Failf(t, "Error writing test config.json: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
config := Configure()
|
||||||
|
assert.Contains(t, config.AllowedWebmentionSources, "snoopy.be")
|
||||||
|
assert.Equal(t, 1, len(config.AllowedWebmentionSources))
|
||||||
|
os.Remove("config.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
|
||||||
|
config := Configure()
|
||||||
|
config.AddToBlacklist("somethingnew.be")
|
||||||
|
config.Save()
|
||||||
|
|
||||||
|
newConfig := Configure()
|
||||||
|
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
|
||||||
|
os.Remove("config.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddToBlacklistNotYetAddsToList(t *testing.T) {
|
||||||
|
conf := Config{
|
||||||
|
Blacklist: []string{
|
||||||
|
"youtube.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
conf.AddToBlacklist("dinges.be")
|
||||||
|
assert.Contains(t, conf.Blacklist, "dinges.be")
|
||||||
|
assert.Equal(t, 2, len(conf.Blacklist))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
|
||||||
|
conf := Config{
|
||||||
|
Blacklist: []string{
|
||||||
|
"youtube.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
conf.AddToBlacklist("youtube.com")
|
||||||
|
assert.Contains(t, conf.Blacklist, "youtube.com")
|
||||||
|
assert.Equal(t, 1, len(conf.Blacklist))
|
||||||
|
}
|
||||||
|
|
||||||
func TestIsBlacklisted(t *testing.T) {
|
func TestIsBlacklisted(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
label string
|
label string
|
||||||
|
|
15
db/repo.go
15
db/repo.go
|
@ -9,6 +9,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
"github.com/tidwall/buntdb"
|
"github.com/tidwall/buntdb"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
type MentionRepoBunt struct {
|
type MentionRepoBunt struct {
|
||||||
|
@ -19,6 +20,7 @@ type MentionRepo interface {
|
||||||
Save(key mf.Mention, data *mf.IndiewebData) (string, error)
|
Save(key mf.Mention, data *mf.IndiewebData) (string, error)
|
||||||
SavePicture(bytes string, domain string) (string, error)
|
SavePicture(bytes string, domain string) (string, error)
|
||||||
Delete(key mf.Mention)
|
Delete(key mf.Mention)
|
||||||
|
CleanupSpam(domain string, blacklist []string)
|
||||||
LastSentMention(domain string) string
|
LastSentMention(domain string) string
|
||||||
UpdateLastSentMention(domain string, lastSent string)
|
UpdateLastSentMention(domain string, lastSent string)
|
||||||
Get(key mf.Mention) *mf.IndiewebData
|
Get(key mf.Mention) *mf.IndiewebData
|
||||||
|
@ -26,6 +28,17 @@ type MentionRepo interface {
|
||||||
GetAll(domain string) mf.IndiewebDataResult
|
GetAll(domain string) mf.IndiewebDataResult
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry.
|
||||||
|
func (r *MentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
|
||||||
|
for _, mention := range r.GetAll(domain).Data {
|
||||||
|
for _, blacklisted := range blacklist {
|
||||||
|
if strings.Contains(mention.Url, blacklisted) {
|
||||||
|
r.Delete(mention.AsMention())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// UpdateLastSentMention updates the last sent mention link. Logs but ignores errors.
|
// UpdateLastSentMention updates the last sent mention link. Logs but ignores errors.
|
||||||
func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) {
|
func (r *MentionRepoBunt) UpdateLastSentMention(domain string, lastSentMentionLink string) {
|
||||||
err := r.db.Update(func(tx *buntdb.Tx) error {
|
err := r.db.Update(func(tx *buntdb.Tx) error {
|
||||||
|
@ -65,6 +78,8 @@ func (r *MentionRepoBunt) Delete(wm mf.Mention) {
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("key", key).Stringer("wm", wm).Msg("Unable to delete")
|
log.Warn().Err(err).Str("key", key).Stringer("wm", wm).Msg("Unable to delete")
|
||||||
|
} else {
|
||||||
|
log.Debug().Str("key", key).Stringer("wm", wm).Msg("Deleted.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,34 @@ func TestSaveAndGetPicture(t *testing.T) {
|
||||||
assert.Equal(t, data, picDataAfterSave)
|
assert.Equal(t, data, picDataAfterSave)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCleanupSpam(t *testing.T) {
|
||||||
|
db := NewMentionRepo(conf)
|
||||||
|
db.Save(mf.Mention{
|
||||||
|
Source: "https://naar.hier/jup",
|
||||||
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
|
}, &mf.IndiewebData{
|
||||||
|
Name: "lolz",
|
||||||
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
|
Source: "https://naar.hier/jup",
|
||||||
|
Url: "https://naar.hier",
|
||||||
|
})
|
||||||
|
db.Save(mf.Mention{
|
||||||
|
Source: "https://spam.be/malicious",
|
||||||
|
Target: "https://pussycat.com/dinges",
|
||||||
|
}, &mf.IndiewebData{
|
||||||
|
Name: "kapot",
|
||||||
|
Target: "https://pussycat.com/dinges",
|
||||||
|
Source: "https://spam.be/malicious",
|
||||||
|
Url: "https://spam.be",
|
||||||
|
})
|
||||||
|
|
||||||
|
db.CleanupSpam("pussycat.com", []string{"spam.be", "jaak.com"})
|
||||||
|
|
||||||
|
results := db.GetAll("pussycat.com")
|
||||||
|
assert.Equal(t, 1, len(results.Data))
|
||||||
|
assert.Equal(t, "lolz", results.Data[0].Name)
|
||||||
|
}
|
||||||
|
|
||||||
func TestDelete(t *testing.T) {
|
func TestDelete(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := NewMentionRepo(conf)
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
|
|
24
main.go
24
main.go
|
@ -1,6 +1,7 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"brainbaking.com/go-jamming/common"
|
||||||
"brainbaking.com/go-jamming/db"
|
"brainbaking.com/go-jamming/db"
|
||||||
"flag"
|
"flag"
|
||||||
"os"
|
"os"
|
||||||
|
@ -16,11 +17,13 @@ func main() {
|
||||||
|
|
||||||
verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)")
|
verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)")
|
||||||
migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.")
|
migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.")
|
||||||
|
blacklist := flag.String("blacklist", "", "Blacklist a domain name (also cleans spam from DB)")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
blacklisting := len(*blacklist) > 1
|
||||||
|
|
||||||
// logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack.
|
// logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack.
|
||||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||||
if *verboseFlag || *migrateFlag {
|
if *verboseFlag || *migrateFlag || blacklisting {
|
||||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
||||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||||
}
|
}
|
||||||
|
@ -30,10 +33,29 @@ func main() {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if blacklisting {
|
||||||
|
blacklistDomain(*blacklist)
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
log.Debug().Msg("Let's a go!")
|
log.Debug().Msg("Let's a go!")
|
||||||
app.Start()
|
app.Start()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func blacklistDomain(domain string) {
|
||||||
|
log.Info().Str("domain", domain).Msg("Blacklisting...")
|
||||||
|
config := common.Configure()
|
||||||
|
config.AddToBlacklist(domain)
|
||||||
|
config.Save()
|
||||||
|
|
||||||
|
repo := db.NewMentionRepo(config)
|
||||||
|
for _, domain := range config.AllowedWebmentionSources {
|
||||||
|
repo.CleanupSpam(domain, config.Blacklist)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Blacklist done, exiting.")
|
||||||
|
}
|
||||||
|
|
||||||
func migrate() {
|
func migrate() {
|
||||||
log.Info().Msg("Starting db migration...")
|
log.Info().Msg("Starting db migration...")
|
||||||
db.Migrate()
|
db.Migrate()
|
||||||
|
|
Loading…
Reference in New Issue