From 47af4bb93f8b0cfc8b70820ec08cf2d6eaa66c85 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Sun, 18 Apr 2021 21:31:31 +0200 Subject: [PATCH] decouple db impl from configuration, cleanup dataPath usage --- app/mf/mention.go | 15 +++++++--- app/mf/mention_test.go | 46 +++++++++++++++++++++++++++++ app/webmention/recv/receive_test.go | 2 +- app/webmention/send/send_test.go | 2 +- common/config.go | 7 +++-- db/repo.go | 13 ++++---- db/repo_test.go | 14 ++++----- migrate-db.go | 19 +++++++----- 8 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 app/mf/mention_test.go diff --git a/app/mf/mention.go b/app/mf/mention.go index 5c3080e..31f2b8e 100644 --- a/app/mf/mention.go +++ b/app/mf/mention.go @@ -1,10 +1,10 @@ package mf import ( - "brainbaking.com/go-jamming/common" "crypto/md5" "fmt" "net/url" + "strings" ) // this should be passed along as a value object, not as a pointer @@ -24,9 +24,16 @@ func (wm Mention) String() string { return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target) } -func (wm Mention) Domain(conf *common.Config) string { - domain, _ := conf.FetchDomain(wm.Target) - return domain +// Domain parses the target url to extract the domain as part of the allowed webmention targets. +// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling. +// Assumes http(s) protocol, which should have been validated by now. +func (wm Mention) Domain() string { + withPossibleSubdomain := strings.Split(wm.Target, "/")[2] + split := strings.Split(withPossibleSubdomain, ".") + if len(split) == 2 { + return withPossibleSubdomain // that was the extention, not the subdomain. + } + return fmt.Sprintf("%s.%s", split[1], split[2]) } func (wm Mention) Key() string { diff --git a/app/mf/mention_test.go b/app/mf/mention_test.go new file mode 100644 index 0000000..31367b4 --- /dev/null +++ b/app/mf/mention_test.go @@ -0,0 +1,46 @@ +package mf + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDomainParseFromTarget(t *testing.T) { + cases := []struct { + label string + target string + expected string + }{ + { + "parse from default http domain", + "http://patat.be/frietjes/zijn/lekker", + "patat.be", + }, + { + "parse from default https domain", + "https://frit.be/patatjes/zijn/lekker", + "frit.be", + }, + { + "parse from default https domain with www subdomain", + "https://www.frit.be/patatjes/zijn/lekker", + "frit.be", + }, + { + "parse from default https domain with some random subdomain", + "https://mayonaise.frit.be/patatjes/zijn/lekker", + "frit.be", + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + wm := Mention{ + Source: "source", + Target: tc.target, + } + + assert.Equal(t, tc.expected, wm.Domain()) + }) + } +} diff --git a/app/webmention/recv/receive_test.go b/app/webmention/recv/receive_test.go index 27e5d36..4490561 100644 --- a/app/webmention/recv/receive_test.go +++ b/app/webmention/recv/receive_test.go @@ -19,7 +19,7 @@ var conf = &common.Config{ "jefklakscodex.com", "brainbaking.com", }, - Connection: ":memory:", + ConString: ":memory:", } func TestReceive(t *testing.T) { diff --git a/app/webmention/send/send_test.go b/app/webmention/send/send_test.go index 186c90c..aa1c157 100644 --- a/app/webmention/send/send_test.go +++ b/app/webmention/send/send_test.go @@ -16,7 +16,7 @@ import ( ) var conf = &common.Config{ - Connection: ":memory:", + ConString: ":memory:", AllowedWebmentionSources: []string{ "domain", }, diff --git a/common/config.go b/common/config.go index 98de6ae..c6b039f 100644 --- a/common/config.go +++ b/common/config.go @@ -13,7 +13,7 @@ type Config struct { Token string `json:"token"` UtcOffset int `json:"utcOffset"` DataPath string `json:"dataPath"` - Connection string `json:"conString"` + ConString string `json:"conString"` AllowedWebmentionSources []string `json:"allowedWebmentionSources"` DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"` } @@ -29,6 +29,9 @@ func (c *Config) missingKeys() []string { if c.DataPath == "" { keys = append(keys, "dataPath") } + if c.ConString == "" { + keys = append(keys, "conString") + } if len(c.AllowedWebmentionSources) == 0 { keys = append(keys, "allowedWebmentionSources") } @@ -97,7 +100,7 @@ func defaultConfig() *Config { Token: "miauwkes", UtcOffset: 60, DataPath: "data", - Connection: "data/mentions.db", + ConString: "data/mentions.db", AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"}, DisallowedWebmentionDomains: []string{"youtube.com"}, } diff --git a/db/repo.go b/db/repo.go index 022fb19..4d9436f 100644 --- a/db/repo.go +++ b/db/repo.go @@ -13,8 +13,7 @@ import ( ) type MentionRepoBunt struct { - db *buntdb.DB - conf *common.Config + db *buntdb.DB } type MentionRepo interface { @@ -79,7 +78,7 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er } func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string { - return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain(r.conf)) + return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain()) } // Get returns a single unmarshalled json value based on the mention key. @@ -129,12 +128,10 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult { // It also creates necessary indexes based on the passed domain config. // This panics if it cannot open the db. func NewMentionRepo(c *common.Config) *MentionRepoBunt { - repo := &MentionRepoBunt{ - conf: c, - } - db, err := buntdb.Open(c.Connection) + repo := &MentionRepoBunt{} + db, err := buntdb.Open(c.ConString) if err != nil { - log.Fatal().Str("constr", c.Connection).Msg("new mention repo: cannot open db") + log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db") } repo.db = db diff --git a/db/repo_test.go b/db/repo_test.go index 693fdfc..c963835 100644 --- a/db/repo_test.go +++ b/db/repo_test.go @@ -13,7 +13,7 @@ import ( var ( conf = &common.Config{ - Connection: ":memory:", + ConString: ":memory:", AllowedWebmentionSources: []string{ "pussycat.com", }, @@ -23,7 +23,7 @@ var ( func TestDelete(t *testing.T) { db := NewMentionRepo(conf) wm := mf.Mention{ - Target: "pussycat.com/coolpussy.html", + Target: "https://pussycat.com/coolpussy.html", } db.Save(wm, &mf.IndiewebData{ Name: "lolz", @@ -56,7 +56,7 @@ func TestSinceFirstTimeIsEmptytime(t *testing.T) { func TestGet(t *testing.T) { db := NewMentionRepo(conf) wm := mf.Mention{ - Target: "pussycat.com/coolpussy.html", + Target: "https://pussycat.com/coolpussy.html", } db.Save(wm, &mf.IndiewebData{ Name: "lolz", @@ -69,7 +69,7 @@ func TestGet(t *testing.T) { func BenchmarkMentionRepoBunt_GetAll(b *testing.B) { defer os.Remove("test.db") db := NewMentionRepo(&common.Config{ - Connection: "test.db", + ConString: "test.db", AllowedWebmentionSources: []string{ "pussycat.com", }, @@ -99,7 +99,7 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) { func TestGetAllAndSaveSomeJson(t *testing.T) { db := NewMentionRepo(conf) db.Save(mf.Mention{ - Target: "pussycat.com/coolpussy.html", + Target: "https://pussycat.com/coolpussy.html", }, &mf.IndiewebData{ Name: "lolz", }) @@ -112,12 +112,12 @@ func TestGetAllAndSaveSomeJson(t *testing.T) { func TestGetFiltersBasedOnDomain(t *testing.T) { db := NewMentionRepo(conf) db.Save(mf.Mention{ - Target: "pussycat.com/coolpussy.html", + Target: "https://pussycat.com/coolpussy.html", }, &mf.IndiewebData{ Name: "lolz", }) db.Save(mf.Mention{ - Target: "dingeling.com/dogshateus.html", + Target: "https://dingeling.com/dogshateus.html", }, &mf.IndiewebData{ Name: "amaigat", }) diff --git a/migrate-db.go b/migrate-db.go index 4c85e1f..8b005b6 100644 --- a/migrate-db.go +++ b/migrate-db.go @@ -11,21 +11,24 @@ import ( "os" ) -func mai() { +// Migrate migrates from data/[domain]/md5hash.json files to the new key/value db. +// This is only needed if you've run go-jamming before the db migration. +func Migrate() { cnf := common.Configure() - os.Remove(cnf.Connection) + dataPath := "data" // decoupled from config, change if needed + os.Remove(cnf.ConString) repo := db.NewMentionRepo(cnf) - log.Info().Str("dbconfig", cnf.Connection).Msg("Starting migration...") + log.Info().Str("dbconfig", cnf.ConString).Msg("Starting migration...") for _, domain := range cnf.AllowedWebmentionSources { fmt.Printf("Processing domain %s...\n", domain) - entries, err := os.ReadDir(fmt.Sprintf("%s/%s", cnf.DataPath, domain)) + entries, err := os.ReadDir(fmt.Sprintf("%s/%s", dataPath, domain)) if err != nil { log.Fatal().Err(err).Msg("Error while reading import path") } for _, file := range entries { - filename := fmt.Sprintf("%s/%s/%s", cnf.DataPath, domain, file.Name()) + filename := fmt.Sprintf("%s/%s/%s", dataPath, domain, file.Name()) data, err := ioutil.ReadFile(filename) if err != nil { log.Fatal().Str("file", filename).Err(err).Msg("Error while reading file") @@ -43,9 +46,9 @@ func mai() { } } - log.Info().Str("dbconfig", cnf.Connection).Msg("Checking for since files...") + log.Info().Str("dbconfig", cnf.ConString).Msg("Checking for since files...") for _, domain := range cnf.AllowedWebmentionSources { - since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", cnf.DataPath, domain)) + since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", dataPath, domain)) if err != nil { log.Warn().Str("domain", domain).Msg("No since found, skipping") continue @@ -55,5 +58,5 @@ func mai() { repo.UpdateSince(domain, common.IsoToTime(string(since))) } - log.Info().Str("dbconfig", cnf.Connection).Msg("Done! Check db") + log.Info().Str("dbconfig", cnf.ConString).Msg("Done! Check db") }