Merge pull request 'Inclusive language update (allowlist/denylist).' (#4) from JonathanStreet/go-jamming:main into master
Reviewed-on: #4master
commit
ab835c0457
|
@ -25,10 +25,10 @@ Place a `config.json` file in the same directory that looks like this: (below ar
|
|||
"brainbaking.com",
|
||||
"jefklakscodex.com"
|
||||
],
|
||||
"blacklist": [
|
||||
"denylist": [
|
||||
"youtube.com"
|
||||
],
|
||||
"whitelist": []
|
||||
"allowlist": []
|
||||
}
|
||||
```
|
||||
|
||||
|
@ -36,7 +36,7 @@ Place a `config.json` file in the same directory that looks like this: (below ar
|
|||
- `adminEmail`, the e-mail address to send notificaions to. If absent, will not send out mails. **uses 127.0.0.1:25 postfix** at the moment.
|
||||
- `port`, host: http server params
|
||||
- `token`: see below, used for authentication
|
||||
- `blacklist`/`whitelist`: domains from which we do (NOT) send to or accept mentions from. This is usually the domain of the `source` in the receiving mention. **Note**: the blacklist is also used to block outgoing mentions.
|
||||
- `denylist`/`allowlist`: domains from which we do (NOT) send to or accept mentions from. This is usually the domain of the `source` in the receiving mention. **Note**: the denylist is also used to block outgoing mentions.
|
||||
- `allowedWebmentionSources`: your own domains which go-jamming is able to receive mentions from. This is usually domain of the `target` in the receiving mention.
|
||||
|
||||
If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`.
|
||||
|
|
16
README.md
16
README.md
|
@ -204,23 +204,23 @@ Still, spammers always find a way and sometimes even create fake blog posts with
|
|||
|
||||
### Mentions _in moderation_
|
||||
|
||||
Go-Jamming employs a `whitelist` and `blacklist` system. By default, all mentions end up in a moderation queue, another database that will not pollute the mention db.
|
||||
Go-Jamming employs a `allowlist` and `denylist` system. By default, all mentions end up in a moderation queue, another database that will not pollute the mention db.
|
||||
|
||||
Each mention has to be manually approved. An e-mail to `localhost:25` (a local Postfix) will be sent out with approve/reject links, if configured. Otherwise, the endpoint `/admin/{token}` is the dashboard where you can approve/reject from time to time:
|
||||
|
||||

|
||||
|
||||
Approved mentions will have their domain added to the whitelist. Rejected mentions will have their domain added to the blacklist.
|
||||
Approved mentions will have their domain added to the allowlist. Rejected mentions will have their domain added to the denylist.
|
||||
|
||||
Read more about how spam moderation works at https://brainbaking.com/post/2022/04/fighting-webmention-and-pingback-spam/
|
||||
|
||||
### Manually blacklisting partial domains
|
||||
### Manually adding partial domains to the denylist
|
||||
|
||||
In that case, simply add the domain to the `blacklist` in `config.json`.
|
||||
In that case, simply add the domain to the `denylist` in `config.json`.
|
||||
|
||||
Adding this **manually** will not remove existing spam in your DB! The `-blacklist` flag is there to:
|
||||
Adding this **manually** will not remove existing spam in your DB! The `-denylist` flag is there to:
|
||||
|
||||
1. Automatically add it to the `blacklist` array in the config file;
|
||||
2. Automatically search the DB for all allowed domains for spam from the blacklist and remove it. (Check for string match on the URL)
|
||||
1. Automatically add it to the `denylist` array in the config file;
|
||||
2. Automatically search the DB for all allowed domains for spam from the denylist and remove it. (Check for string match on the URL)
|
||||
|
||||
How to use: `./go-jamming -blacklist annoyingspam.com`. This will exit after the above actions. Then you can simply restart the server with `./go-jamming`.
|
||||
How to use: `./go-jamming -denylist annoyingspam.com`. This will exit after the above actions. Then you can simply restart the server with `./go-jamming`.
|
||||
|
|
|
@ -107,7 +107,7 @@ func HandleGetToApprove(repo db.MentionRepo) http.HandlerFunc {
|
|||
}
|
||||
}
|
||||
|
||||
// HandleApprove approves the Mention (by key in URL) and adds to the whitelist.
|
||||
// HandleApprove approves the Mention (by key in URL) and adds to the allowlist.
|
||||
// Returns 200 OK with approved source/target or 404 if key is invalid.
|
||||
func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
|
||||
tmpl := asTemplate("moderated", moderatedTemplate)
|
||||
|
@ -121,7 +121,7 @@ func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
c.AddToWhitelist(approved.AsMention().SourceDomain())
|
||||
c.AddToAllowlist(approved.AsMention().SourceDomain())
|
||||
err := tmpl.Execute(w, asDashboardModerated("Approved", approved, c))
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
|
@ -130,7 +130,7 @@ func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
|
|||
}
|
||||
}
|
||||
|
||||
// HandleReject rejects the Mention (by key in URL) and adds to the blacklist.
|
||||
// HandleReject rejects the Mention (by key in URL) and adds to the denylist.
|
||||
// Returns 200 OK with rejected source/target or 404 if key is invalid.
|
||||
func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
|
||||
tmpl := asTemplate("moderated", moderatedTemplate)
|
||||
|
@ -144,7 +144,7 @@ func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
|
|||
return
|
||||
}
|
||||
|
||||
c.AddToBlacklist(rejected.AsMention().SourceDomain())
|
||||
c.AddToDenylist(rejected.AsMention().SourceDomain())
|
||||
err := tmpl.Execute(w, asDashboardModerated("Rejected", rejected, c))
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
|
|
|
@ -20,8 +20,8 @@ var (
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"brainbaking.com"},
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{"brainbaking.com"},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{"brainbaking.com"},
|
||||
}
|
||||
repo db.MentionRepo
|
||||
)
|
||||
|
|
|
@ -63,10 +63,10 @@ func (ib *ImportBootstrapper) Import(file string) {
|
|||
Source: wm.Source,
|
||||
Target: wm.Target,
|
||||
}
|
||||
ib.Conf.AddToWhitelist(mention.SourceDomain())
|
||||
ib.Conf.AddToAllowlist(mention.SourceDomain())
|
||||
|
||||
recv.ProcessAuthorPicture(wm)
|
||||
recv.ProcessWhitelistedMention(mention, wm)
|
||||
recv.ProcessAllowlistedMention(mention, wm)
|
||||
}
|
||||
|
||||
log.Info().Msg("All done, enjoy your go-jammed mentions!")
|
||||
|
|
|
@ -16,8 +16,8 @@ var (
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"chrisburnell.com"},
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{"chrisburnell.com"},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{"chrisburnell.com"},
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -51,6 +51,6 @@ func TestImport(t *testing.T) {
|
|||
assert.Equal(t, "", entries.Data[10].Name)
|
||||
assert.Equal(t, "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/", entries.Data[20].Content)
|
||||
|
||||
assert.Contains(t, cnf.Whitelist, "jacky.wtf")
|
||||
assert.Contains(t, cnf.Whitelist, "martymcgui.re")
|
||||
assert.Contains(t, cnf.Allowlist, "jacky.wtf")
|
||||
assert.Contains(t, cnf.Allowlist, "martymcgui.re")
|
||||
}
|
||||
|
|
|
@ -31,7 +31,7 @@ func (wm Mention) TargetDomain() string {
|
|||
return rest.Domain(wm.Target)
|
||||
}
|
||||
|
||||
// SoureceDomain converts the Source to a domain name to be used in whitelisting/blacklisting (See TargetDomain()).
|
||||
// SoureceDomain converts the Source to a domain name to be used in the allowlist/denylist (See TargetDomain()).
|
||||
func (wm Mention) SourceDomain() string {
|
||||
return rest.Domain(wm.Source)
|
||||
}
|
||||
|
|
|
@ -18,8 +18,8 @@ func TestBuildReceivedMsgDoesNotContainApproveLink(t *testing.T) {
|
|||
},
|
||||
BaseURL: "https://jam.brainbaking.com/",
|
||||
Token: "mytoken",
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{},
|
||||
}
|
||||
|
||||
result := buildReceivedMsg(wm, &mf.IndiewebData{Content: "somecontent"}, cnf)
|
||||
|
@ -40,8 +40,8 @@ func TestBuildInModerationMsgContainsApproveLink(t *testing.T) {
|
|||
},
|
||||
BaseURL: "https://jam.brainbaking.com/",
|
||||
Token: "mytoken",
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{},
|
||||
}
|
||||
|
||||
result := buildInModerationMsg(wm, &mf.IndiewebData{Content: "somecontent"}, cnf)
|
||||
|
|
|
@ -20,8 +20,8 @@ var (
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"brainbaking.com"},
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{"brainbaking.com"},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{"brainbaking.com"},
|
||||
}
|
||||
repo db.MentionRepo
|
||||
)
|
||||
|
|
|
@ -27,8 +27,8 @@ var (
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"brainbaking.com"},
|
||||
Blacklist: []string{"youtube.com"},
|
||||
Whitelist: []string{"brainbaking.com"},
|
||||
Denylist: []string{"youtube.com"},
|
||||
Allowlist: []string{"brainbaking.com"},
|
||||
}
|
||||
repo db.MentionRepo
|
||||
)
|
||||
|
|
|
@ -32,8 +32,8 @@ var (
|
|||
)
|
||||
|
||||
func (recv *Receiver) Receive(wm mf.Mention) {
|
||||
if recv.Conf.IsBlacklisted(wm.Source) {
|
||||
log.Warn().Stringer("wm", wm).Msg(" ABORT: source url comes from blacklisted domain!")
|
||||
if recv.Conf.IsDenylisted(wm.Source) {
|
||||
log.Warn().Stringer("wm", wm).Msg(" ABORT: source url comes from denylisted domain!")
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -64,8 +64,8 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {
|
|||
indieweb := recv.convertBodyToIndiewebData(body, wm, data)
|
||||
recv.ProcessAuthorPicture(indieweb)
|
||||
|
||||
if recv.Conf.IsWhitelisted(wm.Source) {
|
||||
recv.ProcessWhitelistedMention(wm, indieweb)
|
||||
if recv.Conf.IsAllowlisted(wm.Source) {
|
||||
recv.ProcessAllowlistedMention(wm, indieweb)
|
||||
} else {
|
||||
recv.ProcessMentionInModeration(wm, indieweb)
|
||||
}
|
||||
|
@ -83,7 +83,7 @@ func (recv *Receiver) ProcessMentionInModeration(wm mf.Mention, indieweb *mf.Ind
|
|||
log.Info().Str("key", key).Msg("OK: Webmention processed, in moderation.")
|
||||
}
|
||||
|
||||
func (recv *Receiver) ProcessWhitelistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
|
||||
func (recv *Receiver) ProcessAllowlistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
|
||||
key, err := recv.Repo.Save(wm, indieweb)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db")
|
||||
|
@ -92,7 +92,7 @@ func (recv *Receiver) ProcessWhitelistedMention(wm mf.Mention, indieweb *mf.Indi
|
|||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to notify")
|
||||
}
|
||||
log.Info().Str("key", key).Msg("OK: Webmention processed, in whitelist.")
|
||||
log.Info().Str("key", key).Msg("OK: Webmention processed, in allowlist.")
|
||||
}
|
||||
|
||||
func (recv *Receiver) ProcessAuthorPicture(indieweb *mf.IndiewebData) {
|
||||
|
|
|
@ -24,10 +24,10 @@ var conf = &common.Config{
|
|||
"jefklakscodex.com",
|
||||
"brainbaking.com",
|
||||
},
|
||||
Blacklist: []string{
|
||||
"blacklisted.com",
|
||||
Denylist: []string{
|
||||
"denylisted.com",
|
||||
},
|
||||
Whitelist: []string{
|
||||
Allowlist: []string{
|
||||
"brainbaking.com",
|
||||
"jefklakscodex.com",
|
||||
},
|
||||
|
@ -251,7 +251,7 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi
|
|||
assert.Empty(t, indb)
|
||||
}
|
||||
|
||||
func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
|
||||
func TestReceiveFromNotInAllowlistSavesInModerationAndNotifies(t *testing.T) {
|
||||
wm := mf.Mention{
|
||||
Source: "https://brainbaking.com/valid-indieweb-source.html",
|
||||
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
||||
|
@ -262,8 +262,8 @@ func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
|
|||
},
|
||||
BaseURL: "https://jam.brainbaking.com/",
|
||||
Token: "mytoken",
|
||||
Blacklist: []string{},
|
||||
Whitelist: []string{},
|
||||
Denylist: []string{},
|
||||
Allowlist: []string{},
|
||||
}
|
||||
repo := db.NewMentionRepo(cnf)
|
||||
t.Cleanup(db.Purge)
|
||||
|
@ -286,9 +286,9 @@ func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
|
|||
assert.Contains(t, notifierMock.Output, "in moderation!")
|
||||
}
|
||||
|
||||
func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) {
|
||||
func TestReceiveFromDenylistedDomainDoesNothing(t *testing.T) {
|
||||
wm := mf.Mention{
|
||||
Source: "https://blacklisted.com/whoops",
|
||||
Source: "https://denylisted.com/whoops",
|
||||
Target: "https://brainbaking.com/valid-indieweb-source.html",
|
||||
}
|
||||
|
||||
|
@ -382,7 +382,7 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *
|
|||
AllowedWebmentionSources: []string{
|
||||
"brainbaking.com",
|
||||
},
|
||||
Whitelist: []string{
|
||||
Allowlist: []string{
|
||||
"brid.gy",
|
||||
},
|
||||
}
|
||||
|
|
|
@ -65,7 +65,7 @@ func (snder *Sender) collectUniqueHrefsFromHtml(html string) []string {
|
|||
|
||||
for _, match := range hrefRegexp.FindAllStringSubmatch(html, -1) {
|
||||
url := match[1] // [0] is the match of the entire expression, [1] is the capture group
|
||||
if !extRegexp.MatchString(url) && !snder.Conf.IsBlacklisted(url) && !strings.HasPrefix(url, "#") {
|
||||
if !extRegexp.MatchString(url) && !snder.Conf.IsDenylisted(url) && !strings.HasPrefix(url, "#") {
|
||||
urlmap.Add(url)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,8 +19,8 @@ func TestCollectUniqueHrefsFromHtml(t *testing.T) {
|
|||
[]string{},
|
||||
},
|
||||
{
|
||||
"should not collect blacklisted links",
|
||||
`<html><body><a href="https://www.blacklisted.com/wowo.html">sup</a> and also <a href="/dinges">dinges</a>!</body></html>`,
|
||||
"should not collect denylisted links",
|
||||
`<html><body><a href="https://www.denylisted.com/wowo.html">sup</a> and also <a href="/dinges">dinges</a>!</body></html>`,
|
||||
[]string{
|
||||
"/dinges",
|
||||
},
|
||||
|
@ -59,8 +59,8 @@ func TestCollectUniqueHrefsFromHtml(t *testing.T) {
|
|||
|
||||
s := &Sender{
|
||||
Conf: &common.Config{
|
||||
Blacklist: []string{
|
||||
"blacklisted.com",
|
||||
Denylist: []string{
|
||||
"denylisted.com",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -76,7 +76,7 @@ func TestCollect(t *testing.T) {
|
|||
file, _ := ioutil.ReadFile("../../../mocks/samplerss.xml")
|
||||
snder := &Sender{
|
||||
Conf: &common.Config{
|
||||
Blacklist: []string{
|
||||
Denylist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
},
|
||||
|
|
|
@ -43,8 +43,8 @@ var (
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
||||
Blacklist: []string{"youtube.com"},
|
||||
Whitelist: []string{"brainbaking.com"},
|
||||
Denylist: []string{"youtube.com"},
|
||||
Allowlist: []string{"brainbaking.com"},
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
@ -17,16 +17,16 @@ type Config struct {
|
|||
Port int `json:"port"`
|
||||
Token string `json:"token"`
|
||||
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
||||
Blacklist []string `json:"blacklist"`
|
||||
Whitelist []string `json:"whitelist"`
|
||||
Denylist []string `json:"denylist"`
|
||||
Allowlist []string `json:"allowlist"`
|
||||
}
|
||||
|
||||
func (c *Config) IsBlacklisted(url string) bool {
|
||||
return isListedIn(url, c.Blacklist)
|
||||
func (c *Config) IsDenylisted(url string) bool {
|
||||
return isListedIn(url, c.Denylist)
|
||||
}
|
||||
|
||||
func (c *Config) IsWhitelisted(url string) bool {
|
||||
return isListedIn(url, c.Whitelist)
|
||||
func (c *Config) IsAllowlisted(url string) bool {
|
||||
return isListedIn(url, c.Allowlist)
|
||||
}
|
||||
|
||||
func isListedIn(url string, list []string) bool {
|
||||
|
@ -75,15 +75,15 @@ func Configure() *Config {
|
|||
return c
|
||||
}
|
||||
|
||||
// AddToBlacklist adds the given domain to the blacklist slice and persists to disk.
|
||||
func (c *Config) AddToBlacklist(domain string) {
|
||||
c.Blacklist = addToList(domain, c.Blacklist)
|
||||
// AddToDenylist adds the given domain to the denylist slice and persists to disk.
|
||||
func (c *Config) AddToDenylist(domain string) {
|
||||
c.Denylist = addToList(domain, c.Denylist)
|
||||
c.Save()
|
||||
}
|
||||
|
||||
// AddToWhitelist adds the given domain to the whitelist slice and persists to disk.
|
||||
func (c *Config) AddToWhitelist(domain string) {
|
||||
c.Whitelist = addToList(domain, c.Whitelist)
|
||||
// AddToAllowlist adds the given domain to the allowlist slice and persists to disk.
|
||||
func (c *Config) AddToAllowlist(domain string) {
|
||||
c.Allowlist = addToList(domain, c.Allowlist)
|
||||
c.Save()
|
||||
}
|
||||
|
||||
|
@ -137,8 +137,8 @@ func defaultConfig() *Config {
|
|||
Port: 1337,
|
||||
Token: "miauwkes",
|
||||
AllowedWebmentionSources: []string{"mycooldomain.com", "myotherdomain.com"},
|
||||
Blacklist: []string{"youtube.com"},
|
||||
Whitelist: []string{"mycooldomain.com"},
|
||||
Denylist: []string{"youtube.com"},
|
||||
Allowlist: []string{"mycooldomain.com"},
|
||||
}
|
||||
defaultConfig.Save()
|
||||
return defaultConfig
|
||||
|
|
|
@ -29,7 +29,7 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
|||
"allowedWebmentionSources": [
|
||||
"snoopy.be"
|
||||
],
|
||||
"blacklist": [
|
||||
"denylist": [
|
||||
"youtube.com"
|
||||
]
|
||||
}`
|
||||
|
@ -41,20 +41,20 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
|
|||
assert.Equal(t, 1, len(config.AllowedWebmentionSources))
|
||||
}
|
||||
|
||||
func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
|
||||
func TestSaveAfterAddingANewDenylistEntry(t *testing.T) {
|
||||
t.Cleanup(cleanupConfig)
|
||||
|
||||
config := Configure()
|
||||
config.AddToBlacklist("somethingnew.be")
|
||||
config.AddToDenylist("somethingnew.be")
|
||||
config.Save()
|
||||
|
||||
newConfig := Configure()
|
||||
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
|
||||
assert.Contains(t, newConfig.Denylist, "somethingnew.be")
|
||||
}
|
||||
|
||||
func TestWhitelist(t *testing.T) {
|
||||
func TestAllowlist(t *testing.T) {
|
||||
conf := Config{
|
||||
Whitelist: []string{
|
||||
Allowlist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
BaseURL: "https://jam.brainbaking.com/",
|
||||
|
@ -66,18 +66,18 @@ func TestWhitelist(t *testing.T) {
|
|||
os.Remove("config.json")
|
||||
})
|
||||
|
||||
conf.AddToWhitelist("dinges.be")
|
||||
assert.Contains(t, conf.Whitelist, "dinges.be")
|
||||
assert.Equal(t, 2, len(conf.Whitelist))
|
||||
conf.AddToAllowlist("dinges.be")
|
||||
assert.Contains(t, conf.Allowlist, "dinges.be")
|
||||
assert.Equal(t, 2, len(conf.Allowlist))
|
||||
|
||||
confFromFile := Configure()
|
||||
assert.Contains(t, confFromFile.Whitelist, "dinges.be")
|
||||
assert.Equal(t, 2, len(confFromFile.Whitelist))
|
||||
assert.Contains(t, confFromFile.Allowlist, "dinges.be")
|
||||
assert.Equal(t, 2, len(confFromFile.Allowlist))
|
||||
}
|
||||
|
||||
func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) {
|
||||
func TestAddToDenylistNotYetAddsToListAndSaves(t *testing.T) {
|
||||
conf := Config{
|
||||
Blacklist: []string{
|
||||
Denylist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
BaseURL: "https://jam.brainbaking.com/",
|
||||
|
@ -89,18 +89,18 @@ func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) {
|
|||
os.Remove("config.json")
|
||||
})
|
||||
|
||||
conf.AddToBlacklist("dinges.be")
|
||||
assert.Contains(t, conf.Blacklist, "dinges.be")
|
||||
assert.Equal(t, 2, len(conf.Blacklist))
|
||||
conf.AddToDenylist("dinges.be")
|
||||
assert.Contains(t, conf.Denylist, "dinges.be")
|
||||
assert.Equal(t, 2, len(conf.Denylist))
|
||||
|
||||
confFromFile := Configure()
|
||||
assert.Contains(t, confFromFile.Blacklist, "dinges.be")
|
||||
assert.Equal(t, 2, len(confFromFile.Blacklist))
|
||||
assert.Contains(t, confFromFile.Denylist, "dinges.be")
|
||||
assert.Equal(t, 2, len(confFromFile.Denylist))
|
||||
}
|
||||
|
||||
func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
|
||||
func TestAddToDenylistAlreadyAddedDoNotAddAgain(t *testing.T) {
|
||||
conf := Config{
|
||||
Blacklist: []string{
|
||||
Denylist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
Port: 123,
|
||||
|
@ -111,87 +111,87 @@ func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
|
|||
os.Remove("config.json")
|
||||
})
|
||||
|
||||
conf.AddToBlacklist("youtube.com")
|
||||
assert.Contains(t, conf.Blacklist, "youtube.com")
|
||||
assert.Equal(t, 1, len(conf.Blacklist))
|
||||
conf.AddToDenylist("youtube.com")
|
||||
assert.Contains(t, conf.Denylist, "youtube.com")
|
||||
assert.Equal(t, 1, len(conf.Denylist))
|
||||
}
|
||||
|
||||
func TestIsWhitelisted(t *testing.T) {
|
||||
func TestIsAllowlisted(t *testing.T) {
|
||||
cases := []struct {
|
||||
label string
|
||||
url string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"do not whitelist if domain is part of relative url",
|
||||
"do not allowlist if domain is part of relative url",
|
||||
"https://brainbaking.com/post/youtube.com-sucks",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"whitelist if https domain is on the list",
|
||||
"allowlist if https domain is on the list",
|
||||
"https://youtube.com/stuff",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"whitelist if http domain is on the list",
|
||||
"allowlist if http domain is on the list",
|
||||
"http://youtube.com/stuff",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"do not whitelist if relative url",
|
||||
"do not allowlist if relative url",
|
||||
"/youtube.com",
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
conf := Config{
|
||||
Whitelist: []string{
|
||||
Allowlist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.label, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, conf.IsWhitelisted(tc.url))
|
||||
assert.Equal(t, tc.expected, conf.IsAllowlisted(tc.url))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBlacklisted(t *testing.T) {
|
||||
func TestIsDenylisted(t *testing.T) {
|
||||
cases := []struct {
|
||||
label string
|
||||
url string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"do not blacklist if domain is part of relative url",
|
||||
"do not denylist if domain is part of relative url",
|
||||
"https://brainbaking.com/post/youtube.com-sucks",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"blacklist if https domain is on the list",
|
||||
"denylist if https domain is on the list",
|
||||
"https://youtube.com/stuff",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"blacklist if http domain is on the list",
|
||||
"denylist if http domain is on the list",
|
||||
"http://youtube.com/stuff",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"do not blacklist if relative url",
|
||||
"do not denylist if relative url",
|
||||
"/youtube.com",
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
conf := Config{
|
||||
Blacklist: []string{
|
||||
Denylist: []string{
|
||||
"youtube.com",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.label, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, conf.IsBlacklisted(tc.url))
|
||||
assert.Equal(t, tc.expected, conf.IsDenylisted(tc.url))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,11 +15,11 @@ type mentionRepoBunt struct {
|
|||
db *buntdb.DB
|
||||
}
|
||||
|
||||
// CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry.
|
||||
func (r *mentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
|
||||
// CleanupSpam removes potential denylisted spam from the webmention database by checking the url of each entry.
|
||||
func (r *mentionRepoBunt) CleanupSpam(domain string, denylist []string) {
|
||||
for _, mention := range r.GetAll(domain).Data {
|
||||
for _, blacklisted := range blacklist {
|
||||
if strings.Contains(mention.Url, blacklisted) {
|
||||
for _, denylisted := range denylist {
|
||||
if strings.Contains(mention.Url, denylisted) {
|
||||
r.Delete(mention.AsMention())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -31,8 +31,8 @@ type MentionRepo interface {
|
|||
// GetAll returns a wrapped data result for all to approve mentions for a particular domain.
|
||||
GetAllToModerate(domain string) mf.IndiewebDataResult
|
||||
|
||||
// CleanupSpam removes potential blacklisted spam from the approved database by checking the url of each entry.
|
||||
CleanupSpam(domain string, blacklist []string)
|
||||
// CleanupSpam removes potential denylisted spam from the approved database by checking the url of each entry.
|
||||
CleanupSpam(domain string, denylist []string)
|
||||
|
||||
// SavePicture saves the picture byte data in the approved database and returns a key or error.
|
||||
SavePicture(bytes string, domain string) (string, error)
|
||||
|
@ -80,8 +80,8 @@ func (m MentionRepoWrapper) Reject(keyInModeration string) *mf.IndiewebData {
|
|||
return toReject
|
||||
}
|
||||
|
||||
func (m MentionRepoWrapper) CleanupSpam(domain string, blacklist []string) {
|
||||
m.approvedRepo.CleanupSpam(domain, blacklist)
|
||||
func (m MentionRepoWrapper) CleanupSpam(domain string, denylist []string) {
|
||||
m.approvedRepo.CleanupSpam(domain, denylist)
|
||||
}
|
||||
|
||||
func (m MentionRepoWrapper) LastSentMention(domain string) string {
|
||||
|
|
20
main.go
20
main.go
|
@ -19,15 +19,15 @@ func main() {
|
|||
|
||||
verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)")
|
||||
migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.")
|
||||
blacklist := flag.String("blacklist", "", "Blacklist a domain name (also cleans spam from DB)")
|
||||
denylist := flag.String("denylist", "", "Denylist a domain name (also cleans spam from DB)")
|
||||
importFile := flag.String("import", "", "Import mentions from an external source (i.e. webmention.io)")
|
||||
flag.Parse()
|
||||
blacklisting := len(*blacklist) > 1
|
||||
denylisting := len(*denylist) > 1
|
||||
importing := len(*importFile) > 1
|
||||
|
||||
// logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack.
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
if *verboseFlag || *migrateFlag || blacklisting || importing {
|
||||
if *verboseFlag || *migrateFlag || denylisting || importing {
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||
}
|
||||
|
@ -37,8 +37,8 @@ func main() {
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
if blacklisting {
|
||||
blacklistDomain(*blacklist)
|
||||
if denylisting {
|
||||
denylistDomain(*denylist)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
|
@ -63,17 +63,17 @@ func importWebmentionFile(file string) {
|
|||
bootstrapper.Import(file)
|
||||
}
|
||||
|
||||
func blacklistDomain(domain string) {
|
||||
log.Info().Str("domain", domain).Msg("Blacklisting...")
|
||||
func denylistDomain(domain string) {
|
||||
log.Info().Str("domain", domain).Msg("Denylisting...")
|
||||
config := common.Configure()
|
||||
config.AddToBlacklist(domain)
|
||||
config.AddToDenylist(domain)
|
||||
|
||||
repo := db.NewMentionRepo(config)
|
||||
for _, domain := range config.AllowedWebmentionSources {
|
||||
repo.CleanupSpam(domain, config.Blacklist)
|
||||
repo.CleanupSpam(domain, config.Denylist)
|
||||
}
|
||||
|
||||
log.Info().Msg("Blacklist done, exiting.")
|
||||
log.Info().Msg("Denylist done, exiting.")
|
||||
}
|
||||
|
||||
func migrate() {
|
||||
|
|
Loading…
Reference in New Issue