decouple db impl from configuration, cleanup dataPath usage
This commit is contained in:
parent
8cd3cb1f1e
commit
47af4bb93f
|
@ -1,10 +1,10 @@
|
|||
package mf
|
||||
|
||||
import (
|
||||
"brainbaking.com/go-jamming/common"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// this should be passed along as a value object, not as a pointer
|
||||
|
@ -24,9 +24,16 @@ func (wm Mention) String() string {
|
|||
return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
|
||||
}
|
||||
|
||||
func (wm Mention) Domain(conf *common.Config) string {
|
||||
domain, _ := conf.FetchDomain(wm.Target)
|
||||
return domain
|
||||
// Domain parses the target url to extract the domain as part of the allowed webmention targets.
|
||||
// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling.
|
||||
// Assumes http(s) protocol, which should have been validated by now.
|
||||
func (wm Mention) Domain() string {
|
||||
withPossibleSubdomain := strings.Split(wm.Target, "/")[2]
|
||||
split := strings.Split(withPossibleSubdomain, ".")
|
||||
if len(split) == 2 {
|
||||
return withPossibleSubdomain // that was the extention, not the subdomain.
|
||||
}
|
||||
return fmt.Sprintf("%s.%s", split[1], split[2])
|
||||
}
|
||||
|
||||
func (wm Mention) Key() string {
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
package mf
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDomainParseFromTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
label string
|
||||
target string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
"parse from default http domain",
|
||||
"http://patat.be/frietjes/zijn/lekker",
|
||||
"patat.be",
|
||||
},
|
||||
{
|
||||
"parse from default https domain",
|
||||
"https://frit.be/patatjes/zijn/lekker",
|
||||
"frit.be",
|
||||
},
|
||||
{
|
||||
"parse from default https domain with www subdomain",
|
||||
"https://www.frit.be/patatjes/zijn/lekker",
|
||||
"frit.be",
|
||||
},
|
||||
{
|
||||
"parse from default https domain with some random subdomain",
|
||||
"https://mayonaise.frit.be/patatjes/zijn/lekker",
|
||||
"frit.be",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.label, func(t *testing.T) {
|
||||
wm := Mention{
|
||||
Source: "source",
|
||||
Target: tc.target,
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.expected, wm.Domain())
|
||||
})
|
||||
}
|
||||
}
|
|
@ -19,7 +19,7 @@ var conf = &common.Config{
|
|||
"jefklakscodex.com",
|
||||
"brainbaking.com",
|
||||
},
|
||||
Connection: ":memory:",
|
||||
ConString: ":memory:",
|
||||
}
|
||||
|
||||
func TestReceive(t *testing.T) {
|
||||
|
|
|
@ -16,7 +16,7 @@ import (
|
|||
)
|
||||
|
||||
var conf = &common.Config{
|
||||
Connection: ":memory:",
|
||||
ConString: ":memory:",
|
||||
AllowedWebmentionSources: []string{
|
||||
"domain",
|
||||
},
|
||||
|
|
|
@ -13,7 +13,7 @@ type Config struct {
|
|||
Token string `json:"token"`
|
||||
UtcOffset int `json:"utcOffset"`
|
||||
DataPath string `json:"dataPath"`
|
||||
Connection string `json:"conString"`
|
||||
ConString string `json:"conString"`
|
||||
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
||||
DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
|
||||
}
|
||||
|
@ -29,6 +29,9 @@ func (c *Config) missingKeys() []string {
|
|||
if c.DataPath == "" {
|
||||
keys = append(keys, "dataPath")
|
||||
}
|
||||
if c.ConString == "" {
|
||||
keys = append(keys, "conString")
|
||||
}
|
||||
if len(c.AllowedWebmentionSources) == 0 {
|
||||
keys = append(keys, "allowedWebmentionSources")
|
||||
}
|
||||
|
@ -97,7 +100,7 @@ func defaultConfig() *Config {
|
|||
Token: "miauwkes",
|
||||
UtcOffset: 60,
|
||||
DataPath: "data",
|
||||
Connection: "data/mentions.db",
|
||||
ConString: "data/mentions.db",
|
||||
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
||||
DisallowedWebmentionDomains: []string{"youtube.com"},
|
||||
}
|
||||
|
|
13
db/repo.go
13
db/repo.go
|
@ -13,8 +13,7 @@ import (
|
|||
)
|
||||
|
||||
type MentionRepoBunt struct {
|
||||
db *buntdb.DB
|
||||
conf *common.Config
|
||||
db *buntdb.DB
|
||||
}
|
||||
|
||||
type MentionRepo interface {
|
||||
|
@ -79,7 +78,7 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er
|
|||
}
|
||||
|
||||
func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string {
|
||||
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain(r.conf))
|
||||
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
|
||||
}
|
||||
|
||||
// Get returns a single unmarshalled json value based on the mention key.
|
||||
|
@ -129,12 +128,10 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
|
|||
// It also creates necessary indexes based on the passed domain config.
|
||||
// This panics if it cannot open the db.
|
||||
func NewMentionRepo(c *common.Config) *MentionRepoBunt {
|
||||
repo := &MentionRepoBunt{
|
||||
conf: c,
|
||||
}
|
||||
db, err := buntdb.Open(c.Connection)
|
||||
repo := &MentionRepoBunt{}
|
||||
db, err := buntdb.Open(c.ConString)
|
||||
if err != nil {
|
||||
log.Fatal().Str("constr", c.Connection).Msg("new mention repo: cannot open db")
|
||||
log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db")
|
||||
}
|
||||
repo.db = db
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
|
||||
var (
|
||||
conf = &common.Config{
|
||||
Connection: ":memory:",
|
||||
ConString: ":memory:",
|
||||
AllowedWebmentionSources: []string{
|
||||
"pussycat.com",
|
||||
},
|
||||
|
@ -23,7 +23,7 @@ var (
|
|||
func TestDelete(t *testing.T) {
|
||||
db := NewMentionRepo(conf)
|
||||
wm := mf.Mention{
|
||||
Target: "pussycat.com/coolpussy.html",
|
||||
Target: "https://pussycat.com/coolpussy.html",
|
||||
}
|
||||
db.Save(wm, &mf.IndiewebData{
|
||||
Name: "lolz",
|
||||
|
@ -56,7 +56,7 @@ func TestSinceFirstTimeIsEmptytime(t *testing.T) {
|
|||
func TestGet(t *testing.T) {
|
||||
db := NewMentionRepo(conf)
|
||||
wm := mf.Mention{
|
||||
Target: "pussycat.com/coolpussy.html",
|
||||
Target: "https://pussycat.com/coolpussy.html",
|
||||
}
|
||||
db.Save(wm, &mf.IndiewebData{
|
||||
Name: "lolz",
|
||||
|
@ -69,7 +69,7 @@ func TestGet(t *testing.T) {
|
|||
func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
||||
defer os.Remove("test.db")
|
||||
db := NewMentionRepo(&common.Config{
|
||||
Connection: "test.db",
|
||||
ConString: "test.db",
|
||||
AllowedWebmentionSources: []string{
|
||||
"pussycat.com",
|
||||
},
|
||||
|
@ -99,7 +99,7 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
|||
func TestGetAllAndSaveSomeJson(t *testing.T) {
|
||||
db := NewMentionRepo(conf)
|
||||
db.Save(mf.Mention{
|
||||
Target: "pussycat.com/coolpussy.html",
|
||||
Target: "https://pussycat.com/coolpussy.html",
|
||||
}, &mf.IndiewebData{
|
||||
Name: "lolz",
|
||||
})
|
||||
|
@ -112,12 +112,12 @@ func TestGetAllAndSaveSomeJson(t *testing.T) {
|
|||
func TestGetFiltersBasedOnDomain(t *testing.T) {
|
||||
db := NewMentionRepo(conf)
|
||||
db.Save(mf.Mention{
|
||||
Target: "pussycat.com/coolpussy.html",
|
||||
Target: "https://pussycat.com/coolpussy.html",
|
||||
}, &mf.IndiewebData{
|
||||
Name: "lolz",
|
||||
})
|
||||
db.Save(mf.Mention{
|
||||
Target: "dingeling.com/dogshateus.html",
|
||||
Target: "https://dingeling.com/dogshateus.html",
|
||||
}, &mf.IndiewebData{
|
||||
Name: "amaigat",
|
||||
})
|
||||
|
|
|
@ -11,21 +11,24 @@ import (
|
|||
"os"
|
||||
)
|
||||
|
||||
func mai() {
|
||||
// Migrate migrates from data/[domain]/md5hash.json files to the new key/value db.
|
||||
// This is only needed if you've run go-jamming before the db migration.
|
||||
func Migrate() {
|
||||
cnf := common.Configure()
|
||||
os.Remove(cnf.Connection)
|
||||
dataPath := "data" // decoupled from config, change if needed
|
||||
os.Remove(cnf.ConString)
|
||||
repo := db.NewMentionRepo(cnf)
|
||||
|
||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Starting migration...")
|
||||
log.Info().Str("dbconfig", cnf.ConString).Msg("Starting migration...")
|
||||
for _, domain := range cnf.AllowedWebmentionSources {
|
||||
fmt.Printf("Processing domain %s...\n", domain)
|
||||
entries, err := os.ReadDir(fmt.Sprintf("%s/%s", cnf.DataPath, domain))
|
||||
entries, err := os.ReadDir(fmt.Sprintf("%s/%s", dataPath, domain))
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("Error while reading import path")
|
||||
}
|
||||
|
||||
for _, file := range entries {
|
||||
filename := fmt.Sprintf("%s/%s/%s", cnf.DataPath, domain, file.Name())
|
||||
filename := fmt.Sprintf("%s/%s/%s", dataPath, domain, file.Name())
|
||||
data, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
log.Fatal().Str("file", filename).Err(err).Msg("Error while reading file")
|
||||
|
@ -43,9 +46,9 @@ func mai() {
|
|||
}
|
||||
}
|
||||
|
||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Checking for since files...")
|
||||
log.Info().Str("dbconfig", cnf.ConString).Msg("Checking for since files...")
|
||||
for _, domain := range cnf.AllowedWebmentionSources {
|
||||
since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", cnf.DataPath, domain))
|
||||
since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", dataPath, domain))
|
||||
if err != nil {
|
||||
log.Warn().Str("domain", domain).Msg("No since found, skipping")
|
||||
continue
|
||||
|
@ -55,5 +58,5 @@ func mai() {
|
|||
repo.UpdateSince(domain, common.IsoToTime(string(since)))
|
||||
}
|
||||
|
||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Done! Check db")
|
||||
log.Info().Str("dbconfig", cnf.ConString).Msg("Done! Check db")
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue