wgroeneveld
/
go-jamming
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge pull request 'Inclusive language update (allowlist/denylist).' (#4) from JonathanStreet/go-jamming:main into master 

Reviewed-on: #4
master
Wouter Groeneveld 2023-03-03 16:18:01 +01:00
parent 63f410d106 267ce249c1
commit ab835c0457
20 changed files with 124 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
INSTALL.md
View File

@ -25,10 +25,10 @@ Place a `config.json` file in the same directory that looks like this: (below ar
"brainbaking.com",
"jefklakscodex.com"
],
"blacklist": [
"denylist": [
"youtube.com"
],
"whitelist": []
"allowlist": []
}
```
@ -36,7 +36,7 @@ Place a `config.json` file in the same directory that looks like this: (below ar
- `adminEmail`, the e-mail address to send notificaions to. If absent, will not send out mails. **uses 127.0.0.1:25 postfix** at the moment.
- `port`, host: http server params
- `token`: see below, used for authentication
- `blacklist`/`whitelist`: domains from which we do (NOT) send to or accept mentions from. This is usually the domain of the `source` in the receiving mention. **Note**: the blacklist is also used to block outgoing mentions.
- `denylist`/`allowlist`: domains from which we do (NOT) send to or accept mentions from. This is usually the domain of the `source` in the receiving mention. **Note**: the denylist is also used to block outgoing mentions.
- `allowedWebmentionSources`: your own domains which go-jamming is able to receive mentions from. This is usually domain of the `target` in the receiving mention.
If a config file is missing, or required keys are missing, a warning will be generated and default values will be used instead. See `common/config.go`.

16
README.md
View File

@ -204,23 +204,23 @@ Still, spammers always find a way and sometimes even create fake blog posts with
### Mentions _in moderation_
Go-Jamming employs a `whitelist` and `blacklist` system. By default, all mentions end up in a moderation queue, another database that will not pollute the mention db.
Go-Jamming employs a `allowlist` and `denylist` system. By default, all mentions end up in a moderation queue, another database that will not pollute the mention db.
Each mention has to be manually approved. An e-mail to `localhost:25` (a local Postfix) will be sent out with approve/reject links, if configured. Otherwise, the endpoint `/admin/{token}` is the dashboard where you can approve/reject from time to time:
![](https://raw.githubusercontent.com/wgroeneveld/go-jamming/master/adminpanel.jpg)
Approved mentions will have their domain added to the whitelist. Rejected mentions will have their domain added to the blacklist.
Approved mentions will have their domain added to the allowlist. Rejected mentions will have their domain added to the denylist.
Read more about how spam moderation works at https://brainbaking.com/post/2022/04/fighting-webmention-and-pingback-spam/
### Manually blacklisting partial domains
### Manually adding partial domains to the denylist
In that case, simply add the domain to the `blacklist` in `config.json`.
In that case, simply add the domain to the `denylist` in `config.json`.
Adding this **manually** will not remove existing spam in your DB! The `-blacklist` flag is there to:
Adding this **manually** will not remove existing spam in your DB! The `-denylist` flag is there to:
1. Automatically add it to the `blacklist` array in the config file;
2. Automatically search the DB for all allowed domains for spam from the blacklist and remove it. (Check for string match on the URL)
1. Automatically add it to the `denylist` array in the config file;
2. Automatically search the DB for all allowed domains for spam from the denylist and remove it. (Check for string match on the URL)
How to use: `./go-jamming -blacklist annoyingspam.com`. This will exit after the above actions. Then you can simply restart the server with `./go-jamming`.
How to use: `./go-jamming -denylist annoyingspam.com`. This will exit after the above actions. Then you can simply restart the server with `./go-jamming`.

8
app/admin/handler.go
View File

@ -107,7 +107,7 @@ func HandleGetToApprove(repo db.MentionRepo) http.HandlerFunc {
}
}
// HandleApprove approves the Mention (by key in URL) and adds to the whitelist.
// HandleApprove approves the Mention (by key in URL) and adds to the allowlist.
// Returns 200 OK with approved source/target or 404 if key is invalid.
func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
tmpl := asTemplate("moderated", moderatedTemplate)
@ -121,7 +121,7 @@ func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
return
}
c.AddToWhitelist(approved.AsMention().SourceDomain())
c.AddToAllowlist(approved.AsMention().SourceDomain())
err := tmpl.Execute(w, asDashboardModerated("Approved", approved, c))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
@ -130,7 +130,7 @@ func HandleApprove(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
}
}
// HandleReject rejects the Mention (by key in URL) and adds to the blacklist.
// HandleReject rejects the Mention (by key in URL) and adds to the denylist.
// Returns 200 OK with rejected source/target or 404 if key is invalid.
func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
tmpl := asTemplate("moderated", moderatedTemplate)
@ -144,7 +144,7 @@ func HandleReject(c *common.Config, repo db.MentionRepo) http.HandlerFunc {
return
}
c.AddToBlacklist(rejected.AsMention().SourceDomain())
c.AddToDenylist(rejected.AsMention().SourceDomain())
err := tmpl.Execute(w, asDashboardModerated("Rejected", rejected, c))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)

4
app/admin/handler_test.go
View File

@ -20,8 +20,8 @@ var (
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"brainbaking.com"},
Blacklist: []string{},
Whitelist: []string{"brainbaking.com"},
Denylist: []string{},
Allowlist: []string{"brainbaking.com"},
}
repo db.MentionRepo
)

4
app/external/importer.go vendored
View File

@ -63,10 +63,10 @@ func (ib *ImportBootstrapper) Import(file string) {
Source: wm.Source,
Target: wm.Target,
}
ib.Conf.AddToWhitelist(mention.SourceDomain())
ib.Conf.AddToAllowlist(mention.SourceDomain())
recv.ProcessAuthorPicture(wm)
recv.ProcessWhitelistedMention(mention, wm)
recv.ProcessAllowlistedMention(mention, wm)
}
log.Info().Msg("All done, enjoy your go-jammed mentions!")

8
app/external/importer_test.go vendored
View File

@ -16,8 +16,8 @@ var (
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"chrisburnell.com"},
Blacklist: []string{},
Whitelist: []string{"chrisburnell.com"},
Denylist: []string{},
Allowlist: []string{"chrisburnell.com"},
}
)
@ -51,6 +51,6 @@ func TestImport(t *testing.T) {
assert.Equal(t, "", entries.Data[10].Name)
assert.Equal(t, "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/", entries.Data[20].Content)
assert.Contains(t, cnf.Whitelist, "jacky.wtf")
assert.Contains(t, cnf.Whitelist, "martymcgui.re")
assert.Contains(t, cnf.Allowlist, "jacky.wtf")
assert.Contains(t, cnf.Allowlist, "martymcgui.re")
}

2
app/mf/mention.go
View File

@ -31,7 +31,7 @@ func (wm Mention) TargetDomain() string {
return rest.Domain(wm.Target)
}
// SoureceDomain converts the Source to a domain name to be used in whitelisting/blacklisting (See TargetDomain()).
// SoureceDomain converts the Source to a domain name to be used in the allowlist/denylist (See TargetDomain()).
func (wm Mention) SourceDomain() string {
return rest.Domain(wm.Source)
}

8
app/notifier/notifier_test.go
View File

@ -18,8 +18,8 @@ func TestBuildReceivedMsgDoesNotContainApproveLink(t *testing.T) {
},
BaseURL: "https://jam.brainbaking.com/",
Token: "mytoken",
Blacklist: []string{},
Whitelist: []string{},
Denylist: []string{},
Allowlist: []string{},
}
result := buildReceivedMsg(wm, &mf.IndiewebData{Content: "somecontent"}, cnf)
@ -40,8 +40,8 @@ func TestBuildInModerationMsgContainsApproveLink(t *testing.T) {
},
BaseURL: "https://jam.brainbaking.com/",
Token: "mytoken",
Blacklist: []string{},
Whitelist: []string{},
Denylist: []string{},
Allowlist: []string{},
}
result := buildInModerationMsg(wm, &mf.IndiewebData{Content: "somecontent"}, cnf)

4
app/rss/handler_test.go
View File

@ -20,8 +20,8 @@ var (
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"brainbaking.com"},
Blacklist: []string{},
Whitelist: []string{"brainbaking.com"},
Denylist: []string{},
Allowlist: []string{"brainbaking.com"},
}
repo db.MentionRepo
)

4
app/webmention/handler_test.go
View File

@ -27,8 +27,8 @@ var (
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"brainbaking.com"},
Blacklist: []string{"youtube.com"},
Whitelist: []string{"brainbaking.com"},
Denylist: []string{"youtube.com"},
Allowlist: []string{"brainbaking.com"},
}
repo db.MentionRepo
)

12
app/webmention/recv/receive.go
View File

@ -32,8 +32,8 @@ var (
)
func (recv *Receiver) Receive(wm mf.Mention) {
if recv.Conf.IsBlacklisted(wm.Source) {
log.Warn().Stringer("wm", wm).Msg(" ABORT: source url comes from blacklisted domain!")
if recv.Conf.IsDenylisted(wm.Source) {
log.Warn().Stringer("wm", wm).Msg(" ABORT: source url comes from denylisted domain!")
return
}
@ -64,8 +64,8 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {
indieweb := recv.convertBodyToIndiewebData(body, wm, data)
recv.ProcessAuthorPicture(indieweb)
if recv.Conf.IsWhitelisted(wm.Source) {
recv.ProcessWhitelistedMention(wm, indieweb)
if recv.Conf.IsAllowlisted(wm.Source) {
recv.ProcessAllowlistedMention(wm, indieweb)
} else {
recv.ProcessMentionInModeration(wm, indieweb)
}
@ -83,7 +83,7 @@ func (recv *Receiver) ProcessMentionInModeration(wm mf.Mention, indieweb *mf.Ind
log.Info().Str("key", key).Msg("OK: Webmention processed, in moderation.")
}
func (recv *Receiver) ProcessWhitelistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
func (recv *Receiver) ProcessAllowlistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
key, err := recv.Repo.Save(wm, indieweb)
if err != nil {
log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db")
@ -92,7 +92,7 @@ func (recv *Receiver) ProcessWhitelistedMention(wm mf.Mention, indieweb *mf.Indi
if err != nil {
log.Error().Err(err).Msg("Failed to notify")
}
log.Info().Str("key", key).Msg("OK: Webmention processed, in whitelist.")
log.Info().Str("key", key).Msg("OK: Webmention processed, in allowlist.")
}
func (recv *Receiver) ProcessAuthorPicture(indieweb *mf.IndiewebData) {

18
app/webmention/recv/receive_test.go
View File

@ -24,10 +24,10 @@ var conf = &common.Config{
"jefklakscodex.com",
"brainbaking.com",
},
Blacklist: []string{
"blacklisted.com",
Denylist: []string{
"denylisted.com",
},
Whitelist: []string{
Allowlist: []string{
"brainbaking.com",
"jefklakscodex.com",
},
@ -251,7 +251,7 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi
assert.Empty(t, indb)
}
func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
func TestReceiveFromNotInAllowlistSavesInModerationAndNotifies(t *testing.T) {
wm := mf.Mention{
Source: "https://brainbaking.com/valid-indieweb-source.html",
Target: "https://brainbaking.com/valid-indieweb-target.html",
@ -262,8 +262,8 @@ func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
},
BaseURL: "https://jam.brainbaking.com/",
Token: "mytoken",
Blacklist: []string{},
Whitelist: []string{},
Denylist: []string{},
Allowlist: []string{},
}
repo := db.NewMentionRepo(cnf)
t.Cleanup(db.Purge)
@ -286,9 +286,9 @@ func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
assert.Contains(t, notifierMock.Output, "in moderation!")
}
func TestReceiveFromBlacklistedDomainDoesNothing(t *testing.T) {
func TestReceiveFromDenylistedDomainDoesNothing(t *testing.T) {
wm := mf.Mention{
Source: "https://blacklisted.com/whoops",
Source: "https://denylisted.com/whoops",
Target: "https://brainbaking.com/valid-indieweb-source.html",
}
@ -382,7 +382,7 @@ func TestProcessSourceBodyAnonymizesBothAuthorPictureAndNameIfComingFromSilo(t *
AllowedWebmentionSources: []string{
"brainbaking.com",
},
Whitelist: []string{
Allowlist: []string{
"brid.gy",
},
}

2
app/webmention/send/rsslinkcollector.go
View File

@ -65,7 +65,7 @@ func (snder *Sender) collectUniqueHrefsFromHtml(html string) []string {
for _, match := range hrefRegexp.FindAllStringSubmatch(html, -1) {
url := match[1] // [0] is the match of the entire expression, [1] is the capture group
if !extRegexp.MatchString(url) && !snder.Conf.IsBlacklisted(url) && !strings.HasPrefix(url, "#") {
if !extRegexp.MatchString(url) && !snder.Conf.IsDenylisted(url) && !strings.HasPrefix(url, "#") {
urlmap.Add(url)
}
}

10
app/webmention/send/rsslinkcollector_test.go
View File

@ -19,8 +19,8 @@ func TestCollectUniqueHrefsFromHtml(t *testing.T) {
[]string{},
},
{
"should not collect blacklisted links",
`<html><body><a href="https://www.blacklisted.com/wowo.html">sup</a> and also <a href="/dinges">dinges</a>!</body></html>`,
"should not collect denylisted links",
`<html><body><a href="https://www.denylisted.com/wowo.html">sup</a> and also <a href="/dinges">dinges</a>!</body></html>`,
[]string{
"/dinges",
},
@ -59,8 +59,8 @@ func TestCollectUniqueHrefsFromHtml(t *testing.T) {
s := &Sender{
Conf: &common.Config{
Blacklist: []string{
"blacklisted.com",
Denylist: []string{
"denylisted.com",
},
},
}
@ -76,7 +76,7 @@ func TestCollect(t *testing.T) {
file, _ := ioutil.ReadFile("../../../mocks/samplerss.xml")
snder := &Sender{
Conf: &common.Config{
Blacklist: []string{
Denylist: []string{
"youtube.com",
},
},

4
app/webmention/validate_test.go
View File

@ -43,8 +43,8 @@ var (
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
Blacklist: []string{"youtube.com"},
Whitelist: []string{"brainbaking.com"},
Denylist: []string{"youtube.com"},
Allowlist: []string{"brainbaking.com"},
}
)

28
common/config.go
View File

@ -17,16 +17,16 @@ type Config struct {
Port int `json:"port"`
Token string `json:"token"`
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
Blacklist []string `json:"blacklist"`
Whitelist []string `json:"whitelist"`
Denylist []string `json:"denylist"`
Allowlist []string `json:"allowlist"`
}
func (c *Config) IsBlacklisted(url string) bool {
return isListedIn(url, c.Blacklist)
func (c *Config) IsDenylisted(url string) bool {
return isListedIn(url, c.Denylist)
}
func (c *Config) IsWhitelisted(url string) bool {
return isListedIn(url, c.Whitelist)
func (c *Config) IsAllowlisted(url string) bool {
return isListedIn(url, c.Allowlist)
}
func isListedIn(url string, list []string) bool {
@ -75,15 +75,15 @@ func Configure() *Config {
return c
}
// AddToBlacklist adds the given domain to the blacklist slice and persists to disk.
func (c *Config) AddToBlacklist(domain string) {
c.Blacklist = addToList(domain, c.Blacklist)
// AddToDenylist adds the given domain to the denylist slice and persists to disk.
func (c *Config) AddToDenylist(domain string) {
c.Denylist = addToList(domain, c.Denylist)
c.Save()
}
// AddToWhitelist adds the given domain to the whitelist slice and persists to disk.
func (c *Config) AddToWhitelist(domain string) {
c.Whitelist = addToList(domain, c.Whitelist)
// AddToAllowlist adds the given domain to the allowlist slice and persists to disk.
func (c *Config) AddToAllowlist(domain string) {
c.Allowlist = addToList(domain, c.Allowlist)
c.Save()
}
@ -137,8 +137,8 @@ func defaultConfig() *Config {
Port: 1337,
Token: "miauwkes",
AllowedWebmentionSources: []string{"mycooldomain.com", "myotherdomain.com"},
Blacklist: []string{"youtube.com"},
Whitelist: []string{"mycooldomain.com"},
Denylist: []string{"youtube.com"},
Allowlist: []string{"mycooldomain.com"},
}
defaultConfig.Save()
return defaultConfig

74
common/config_test.go
View File

@ -29,7 +29,7 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
"allowedWebmentionSources": [
"snoopy.be"
],
"blacklist": [
"denylist": [
"youtube.com"
]
}`
@ -41,20 +41,20 @@ func TestReadFromJsonWithCorrectJsonData(t *testing.T) {
assert.Equal(t, 1, len(config.AllowedWebmentionSources))
}
func TestSaveAfterAddingANewBlacklistEntry(t *testing.T) {
func TestSaveAfterAddingANewDenylistEntry(t *testing.T) {
t.Cleanup(cleanupConfig)
config := Configure()
config.AddToBlacklist("somethingnew.be")
config.AddToDenylist("somethingnew.be")
config.Save()
newConfig := Configure()
assert.Contains(t, newConfig.Blacklist, "somethingnew.be")
assert.Contains(t, newConfig.Denylist, "somethingnew.be")
}
func TestWhitelist(t *testing.T) {
func TestAllowlist(t *testing.T) {
conf := Config{
Whitelist: []string{
Allowlist: []string{
"youtube.com",
},
BaseURL: "https://jam.brainbaking.com/",
@ -66,18 +66,18 @@ func TestWhitelist(t *testing.T) {
os.Remove("config.json")
})
conf.AddToWhitelist("dinges.be")
assert.Contains(t, conf.Whitelist, "dinges.be")
assert.Equal(t, 2, len(conf.Whitelist))
conf.AddToAllowlist("dinges.be")
assert.Contains(t, conf.Allowlist, "dinges.be")
assert.Equal(t, 2, len(conf.Allowlist))
confFromFile := Configure()
assert.Contains(t, confFromFile.Whitelist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Whitelist))
assert.Contains(t, confFromFile.Allowlist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Allowlist))
}
func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) {
func TestAddToDenylistNotYetAddsToListAndSaves(t *testing.T) {
conf := Config{
Blacklist: []string{
Denylist: []string{
"youtube.com",
},
BaseURL: "https://jam.brainbaking.com/",
@ -89,18 +89,18 @@ func TestAddToBlacklistNotYetAddsToListAndSaves(t *testing.T) {
os.Remove("config.json")
})
conf.AddToBlacklist("dinges.be")
assert.Contains(t, conf.Blacklist, "dinges.be")
assert.Equal(t, 2, len(conf.Blacklist))
conf.AddToDenylist("dinges.be")
assert.Contains(t, conf.Denylist, "dinges.be")
assert.Equal(t, 2, len(conf.Denylist))
confFromFile := Configure()
assert.Contains(t, confFromFile.Blacklist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Blacklist))
assert.Contains(t, confFromFile.Denylist, "dinges.be")
assert.Equal(t, 2, len(confFromFile.Denylist))
}
func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
func TestAddToDenylistAlreadyAddedDoNotAddAgain(t *testing.T) {
conf := Config{
Blacklist: []string{
Denylist: []string{
"youtube.com",
},
Port: 123,
@ -111,87 +111,87 @@ func TestAddToBlacklistAlreadyAddedDoNotAddAgain(t *testing.T) {
os.Remove("config.json")
})
conf.AddToBlacklist("youtube.com")
assert.Contains(t, conf.Blacklist, "youtube.com")
assert.Equal(t, 1, len(conf.Blacklist))
conf.AddToDenylist("youtube.com")
assert.Contains(t, conf.Denylist, "youtube.com")
assert.Equal(t, 1, len(conf.Denylist))
}
func TestIsWhitelisted(t *testing.T) {
func TestIsAllowlisted(t *testing.T) {
cases := []struct {
label string
url string
expected bool
}{
{
"do not whitelist if domain is part of relative url",
"do not allowlist if domain is part of relative url",
"https://brainbaking.com/post/youtube.com-sucks",
false,
},
{
"whitelist if https domain is on the list",
"allowlist if https domain is on the list",
"https://youtube.com/stuff",
true,
},
{
"whitelist if http domain is on the list",
"allowlist if http domain is on the list",
"http://youtube.com/stuff",
true,
},
{
"do not whitelist if relative url",
"do not allowlist if relative url",
"/youtube.com",
false,
},
}
conf := Config{
Whitelist: []string{
Allowlist: []string{
"youtube.com",
},
}
for _, tc := range cases {
t.Run(tc.label, func(t *testing.T) {
assert.Equal(t, tc.expected, conf.IsWhitelisted(tc.url))
assert.Equal(t, tc.expected, conf.IsAllowlisted(tc.url))
})
}
}
func TestIsBlacklisted(t *testing.T) {
func TestIsDenylisted(t *testing.T) {
cases := []struct {
label string
url string
expected bool
}{
{
"do not blacklist if domain is part of relative url",
"do not denylist if domain is part of relative url",
"https://brainbaking.com/post/youtube.com-sucks",
false,
},
{
"blacklist if https domain is on the list",
"denylist if https domain is on the list",
"https://youtube.com/stuff",
true,
},
{
"blacklist if http domain is on the list",
"denylist if http domain is on the list",
"http://youtube.com/stuff",
true,
},
{
"do not blacklist if relative url",
"do not denylist if relative url",
"/youtube.com",
false,
},
}
conf := Config{
Blacklist: []string{
Denylist: []string{
"youtube.com",
},
}
for _, tc := range cases {
t.Run(tc.label, func(t *testing.T) {
assert.Equal(t, tc.expected, conf.IsBlacklisted(tc.url))
assert.Equal(t, tc.expected, conf.IsDenylisted(tc.url))
})
}
}

8
db/buntrepo.go
View File

@ -15,11 +15,11 @@ type mentionRepoBunt struct {
db *buntdb.DB
}
// CleanupSpam removes potential blacklisted spam from the webmention database by checking the url of each entry.
func (r *mentionRepoBunt) CleanupSpam(domain string, blacklist []string) {
// CleanupSpam removes potential denylisted spam from the webmention database by checking the url of each entry.
func (r *mentionRepoBunt) CleanupSpam(domain string, denylist []string) {
for _, mention := range r.GetAll(domain).Data {
for _, blacklisted := range blacklist {
if strings.Contains(mention.Url, blacklisted) {
for _, denylisted := range denylist {
if strings.Contains(mention.Url, denylisted) {
r.Delete(mention.AsMention())
}
}

8
db/mentionrepo.go
View File

@ -31,8 +31,8 @@ type MentionRepo interface {
// GetAll returns a wrapped data result for all to approve mentions for a particular domain.
GetAllToModerate(domain string) mf.IndiewebDataResult
// CleanupSpam removes potential blacklisted spam from the approved database by checking the url of each entry.
CleanupSpam(domain string, blacklist []string)
// CleanupSpam removes potential denylisted spam from the approved database by checking the url of each entry.
CleanupSpam(domain string, denylist []string)
// SavePicture saves the picture byte data in the approved database and returns a key or error.
SavePicture(bytes string, domain string) (string, error)
@ -80,8 +80,8 @@ func (m MentionRepoWrapper) Reject(keyInModeration string) *mf.IndiewebData {
return toReject
}
func (m MentionRepoWrapper) CleanupSpam(domain string, blacklist []string) {
m.approvedRepo.CleanupSpam(domain, blacklist)
func (m MentionRepoWrapper) CleanupSpam(domain string, denylist []string) {
m.approvedRepo.CleanupSpam(domain, denylist)
}
func (m MentionRepoWrapper) LastSentMention(domain string) string {

20
main.go
View File

@ -19,15 +19,15 @@ func main() {
verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)")
migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.")
blacklist := flag.String("blacklist", "", "Blacklist a domain name (also cleans spam from DB)")
denylist := flag.String("denylist", "", "Denylist a domain name (also cleans spam from DB)")
importFile := flag.String("import", "", "Import mentions from an external source (i.e. webmention.io)")
flag.Parse()
blacklisting := len(*blacklist) > 1
denylisting := len(*denylist) > 1
importing := len(*importFile) > 1
// logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack.
zerolog.SetGlobalLevel(zerolog.InfoLevel)
if *verboseFlag || *migrateFlag || blacklisting || importing {
if *verboseFlag || *migrateFlag || denylisting || importing {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
@ -37,8 +37,8 @@ func main() {
os.Exit(0)
}
if blacklisting {
blacklistDomain(*blacklist)
if denylisting {
denylistDomain(*denylist)
os.Exit(0)
}
@ -63,17 +63,17 @@ func importWebmentionFile(file string) {
bootstrapper.Import(file)
}
func blacklistDomain(domain string) {
log.Info().Str("domain", domain).Msg("Blacklisting...")
func denylistDomain(domain string) {
log.Info().Str("domain", domain).Msg("Denylisting...")
config := common.Configure()
config.AddToBlacklist(domain)
config.AddToDenylist(domain)
repo := db.NewMentionRepo(config)
for _, domain := range config.AllowedWebmentionSources {
repo.CleanupSpam(domain, config.Blacklist)
repo.CleanupSpam(domain, config.Denylist)
}
log.Info().Msg("Blacklist done, exiting.")
log.Info().Msg("Denylist done, exiting.")
}
func migrate() {