decouple db impl from configuration, cleanup dataPath usage

app/mf/mention.go

@@ -1,10 +1,10 @@
 package mf
 
 import (
-	"brainbaking.com/go-jamming/common"
 	"crypto/md5"
 	"fmt"
 	"net/url"
+	"strings"
 )
 
 // this should be passed along as a value object, not as a pointer
@@ -24,9 +24,16 @@ func (wm Mention) String() string {
 	return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
 }
 
-func (wm Mention) Domain(conf *common.Config) string {
-	domain, _ := conf.FetchDomain(wm.Target)
-	return domain
+// Domain parses the target url to extract the domain as part of the allowed webmention targets.
+// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling.
+// Assumes http(s) protocol, which should have been validated by now.
+func (wm Mention) Domain() string {
+	withPossibleSubdomain := strings.Split(wm.Target, "/")[2]
+	split := strings.Split(withPossibleSubdomain, ".")
+	if len(split) == 2 {
+		return withPossibleSubdomain // that was the extention, not the subdomain.
+	}
+	return fmt.Sprintf("%s.%s", split[1], split[2])
 }
 
 func (wm Mention) Key() string {

app/mf/mention_test.go

@@ -0,0 +1,46 @@
+package mf
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestDomainParseFromTarget(t *testing.T) {
+	cases := []struct {
+		label    string
+		target   string
+		expected string
+	}{
+		{
+			"parse from default http domain",
+			"http://patat.be/frietjes/zijn/lekker",
+			"patat.be",
+		},
+		{
+			"parse from default https domain",
+			"https://frit.be/patatjes/zijn/lekker",
+			"frit.be",
+		},
+		{
+			"parse from default https domain with www subdomain",
+			"https://www.frit.be/patatjes/zijn/lekker",
+			"frit.be",
+		},
+		{
+			"parse from default https domain with some random subdomain",
+			"https://mayonaise.frit.be/patatjes/zijn/lekker",
+			"frit.be",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.label, func(t *testing.T) {
+			wm := Mention{
+				Source: "source",
+				Target: tc.target,
+			}
+
+			assert.Equal(t, tc.expected, wm.Domain())
+		})
+	}
+}

app/webmention/recv/receive_test.go

@@ -19,7 +19,7 @@ var conf = &common.Config{
 		"jefklakscodex.com",
 		"brainbaking.com",
 	},
-	Connection: ":memory:",
+	ConString: ":memory:",
 }
 
 func TestReceive(t *testing.T) {

app/webmention/send/send_test.go

@@ -16,7 +16,7 @@ import (
 )
 
 var conf = &common.Config{
-	Connection: ":memory:",
+	ConString: ":memory:",
 	AllowedWebmentionSources: []string{
 		"domain",
 	},

common/config.go

@@ -13,7 +13,7 @@ type Config struct {
 	Token                       string   `json:"token"`
 	UtcOffset                   int      `json:"utcOffset"`
 	DataPath                    string   `json:"dataPath"`
-	Connection                  string   `json:"conString"`
+	ConString                   string   `json:"conString"`
 	AllowedWebmentionSources    []string `json:"allowedWebmentionSources"`
 	DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
 }
@@ -29,6 +29,9 @@ func (c *Config) missingKeys() []string {
 	if c.DataPath == "" {
 		keys = append(keys, "dataPath")
 	}
+	if c.ConString == "" {
+		keys = append(keys, "conString")
+	}
 	if len(c.AllowedWebmentionSources) == 0 {
 		keys = append(keys, "allowedWebmentionSources")
 	}
@@ -97,7 +100,7 @@ func defaultConfig() *Config {
 		Token:                       "miauwkes",
 		UtcOffset:                   60,
 		DataPath:                    "data",
-		Connection:                  "data/mentions.db",
+		ConString:                   "data/mentions.db",
 		AllowedWebmentionSources:    []string{"brainbaking.com", "jefklakscodex.com"},
 		DisallowedWebmentionDomains: []string{"youtube.com"},
 	}

db/repo.go

@@ -13,8 +13,7 @@ import (
 )
 
 type MentionRepoBunt struct {
-	db   *buntdb.DB
-	conf *common.Config
+	db *buntdb.DB
 }
 
 type MentionRepo interface {
@@ -79,7 +78,7 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er
 }
 
 func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string {
-	return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain(r.conf))
+	return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
 }
 
 // Get returns a single unmarshalled json value based on the mention key.
@@ -129,12 +128,10 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
 // It also creates necessary indexes based on the passed domain config.
 // This panics if it cannot open the db.
 func NewMentionRepo(c *common.Config) *MentionRepoBunt {
-	repo := &MentionRepoBunt{
-		conf: c,
-	}
-	db, err := buntdb.Open(c.Connection)
+	repo := &MentionRepoBunt{}
+	db, err := buntdb.Open(c.ConString)
 	if err != nil {
-		log.Fatal().Str("constr", c.Connection).Msg("new mention repo: cannot open db")
+		log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db")
 	}
 	repo.db = db
 

db/repo_test.go

@@ -13,7 +13,7 @@ import (
 
 var (
 	conf = &common.Config{
-		Connection: ":memory:",
+		ConString: ":memory:",
 		AllowedWebmentionSources: []string{
 			"pussycat.com",
 		},
@@ -23,7 +23,7 @@ var (
 func TestDelete(t *testing.T) {
 	db := NewMentionRepo(conf)
 	wm := mf.Mention{
-		Target: "pussycat.com/coolpussy.html",
+		Target: "https://pussycat.com/coolpussy.html",
 	}
 	db.Save(wm, &mf.IndiewebData{
 		Name: "lolz",
@@ -56,7 +56,7 @@ func TestSinceFirstTimeIsEmptytime(t *testing.T) {
 func TestGet(t *testing.T) {
 	db := NewMentionRepo(conf)
 	wm := mf.Mention{
-		Target: "pussycat.com/coolpussy.html",
+		Target: "https://pussycat.com/coolpussy.html",
 	}
 	db.Save(wm, &mf.IndiewebData{
 		Name: "lolz",
@@ -69,7 +69,7 @@ func TestGet(t *testing.T) {
 func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
 	defer os.Remove("test.db")
 	db := NewMentionRepo(&common.Config{
-		Connection: "test.db",
+		ConString: "test.db",
 		AllowedWebmentionSources: []string{
 			"pussycat.com",
 		},
@@ -99,7 +99,7 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
 func TestGetAllAndSaveSomeJson(t *testing.T) {
 	db := NewMentionRepo(conf)
 	db.Save(mf.Mention{
-		Target: "pussycat.com/coolpussy.html",
+		Target: "https://pussycat.com/coolpussy.html",
 	}, &mf.IndiewebData{
 		Name: "lolz",
 	})
@@ -112,12 +112,12 @@ func TestGetAllAndSaveSomeJson(t *testing.T) {
 func TestGetFiltersBasedOnDomain(t *testing.T) {
 	db := NewMentionRepo(conf)
 	db.Save(mf.Mention{
-		Target: "pussycat.com/coolpussy.html",
+		Target: "https://pussycat.com/coolpussy.html",
 	}, &mf.IndiewebData{
 		Name: "lolz",
 	})
 	db.Save(mf.Mention{
-		Target: "dingeling.com/dogshateus.html",
+		Target: "https://dingeling.com/dogshateus.html",
 	}, &mf.IndiewebData{
 		Name: "amaigat",
 	})

migrate-db.go

@@ -11,21 +11,24 @@ import (
 	"os"
 )
 
-func mai() {
+// Migrate migrates from data/[domain]/md5hash.json files to the new key/value db.
+// This is only needed if you've run go-jamming before the db migration.
+func Migrate() {
 	cnf := common.Configure()
-	os.Remove(cnf.Connection)
+	dataPath := "data" // decoupled from config, change if needed
+	os.Remove(cnf.ConString)
 	repo := db.NewMentionRepo(cnf)
 
-	log.Info().Str("dbconfig", cnf.Connection).Msg("Starting migration...")
+	log.Info().Str("dbconfig", cnf.ConString).Msg("Starting migration...")
 	for _, domain := range cnf.AllowedWebmentionSources {
 		fmt.Printf("Processing domain %s...\n", domain)
-		entries, err := os.ReadDir(fmt.Sprintf("%s/%s", cnf.DataPath, domain))
+		entries, err := os.ReadDir(fmt.Sprintf("%s/%s", dataPath, domain))
 		if err != nil {
 			log.Fatal().Err(err).Msg("Error while reading import path")
 		}
 
 		for _, file := range entries {
-			filename := fmt.Sprintf("%s/%s/%s", cnf.DataPath, domain, file.Name())
+			filename := fmt.Sprintf("%s/%s/%s", dataPath, domain, file.Name())
 			data, err := ioutil.ReadFile(filename)
 			if err != nil {
 				log.Fatal().Str("file", filename).Err(err).Msg("Error while reading file")
@@ -43,9 +46,9 @@ func mai() {
 		}
 	}
 
-	log.Info().Str("dbconfig", cnf.Connection).Msg("Checking for since files...")
+	log.Info().Str("dbconfig", cnf.ConString).Msg("Checking for since files...")
 	for _, domain := range cnf.AllowedWebmentionSources {
-		since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", cnf.DataPath, domain))
+		since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", dataPath, domain))
 		if err != nil {
 			log.Warn().Str("domain", domain).Msg("No since found, skipping")
 			continue
@@ -55,5 +58,5 @@ func mai() {
 		repo.UpdateSince(domain, common.IsoToTime(string(since)))
 	}
 
-	log.Info().Str("dbconfig", cnf.Connection).Msg("Done! Check db")
+	log.Info().Str("dbconfig", cnf.ConString).Msg("Done! Check db")
 }