forked from wgroeneveld/go-jamming
decouple db impl from configuration, cleanup dataPath usage
This commit is contained in:
parent
8cd3cb1f1e
commit
47af4bb93f
|
@ -1,10 +1,10 @@
|
||||||
package mf
|
package mf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"brainbaking.com/go-jamming/common"
|
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// this should be passed along as a value object, not as a pointer
|
// this should be passed along as a value object, not as a pointer
|
||||||
|
@ -24,9 +24,16 @@ func (wm Mention) String() string {
|
||||||
return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
|
return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wm Mention) Domain(conf *common.Config) string {
|
// Domain parses the target url to extract the domain as part of the allowed webmention targets.
|
||||||
domain, _ := conf.FetchDomain(wm.Target)
|
// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling.
|
||||||
return domain
|
// Assumes http(s) protocol, which should have been validated by now.
|
||||||
|
func (wm Mention) Domain() string {
|
||||||
|
withPossibleSubdomain := strings.Split(wm.Target, "/")[2]
|
||||||
|
split := strings.Split(withPossibleSubdomain, ".")
|
||||||
|
if len(split) == 2 {
|
||||||
|
return withPossibleSubdomain // that was the extention, not the subdomain.
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s.%s", split[1], split[2])
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wm Mention) Key() string {
|
func (wm Mention) Key() string {
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
package mf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDomainParseFromTarget(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
label string
|
||||||
|
target string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"parse from default http domain",
|
||||||
|
"http://patat.be/frietjes/zijn/lekker",
|
||||||
|
"patat.be",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parse from default https domain",
|
||||||
|
"https://frit.be/patatjes/zijn/lekker",
|
||||||
|
"frit.be",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parse from default https domain with www subdomain",
|
||||||
|
"https://www.frit.be/patatjes/zijn/lekker",
|
||||||
|
"frit.be",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"parse from default https domain with some random subdomain",
|
||||||
|
"https://mayonaise.frit.be/patatjes/zijn/lekker",
|
||||||
|
"frit.be",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.label, func(t *testing.T) {
|
||||||
|
wm := Mention{
|
||||||
|
Source: "source",
|
||||||
|
Target: tc.target,
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, tc.expected, wm.Domain())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -19,7 +19,7 @@ var conf = &common.Config{
|
||||||
"jefklakscodex.com",
|
"jefklakscodex.com",
|
||||||
"brainbaking.com",
|
"brainbaking.com",
|
||||||
},
|
},
|
||||||
Connection: ":memory:",
|
ConString: ":memory:",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReceive(t *testing.T) {
|
func TestReceive(t *testing.T) {
|
||||||
|
|
|
@ -16,7 +16,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var conf = &common.Config{
|
var conf = &common.Config{
|
||||||
Connection: ":memory:",
|
ConString: ":memory:",
|
||||||
AllowedWebmentionSources: []string{
|
AllowedWebmentionSources: []string{
|
||||||
"domain",
|
"domain",
|
||||||
},
|
},
|
||||||
|
|
|
@ -13,7 +13,7 @@ type Config struct {
|
||||||
Token string `json:"token"`
|
Token string `json:"token"`
|
||||||
UtcOffset int `json:"utcOffset"`
|
UtcOffset int `json:"utcOffset"`
|
||||||
DataPath string `json:"dataPath"`
|
DataPath string `json:"dataPath"`
|
||||||
Connection string `json:"conString"`
|
ConString string `json:"conString"`
|
||||||
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
AllowedWebmentionSources []string `json:"allowedWebmentionSources"`
|
||||||
DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
|
DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
|
||||||
}
|
}
|
||||||
|
@ -29,6 +29,9 @@ func (c *Config) missingKeys() []string {
|
||||||
if c.DataPath == "" {
|
if c.DataPath == "" {
|
||||||
keys = append(keys, "dataPath")
|
keys = append(keys, "dataPath")
|
||||||
}
|
}
|
||||||
|
if c.ConString == "" {
|
||||||
|
keys = append(keys, "conString")
|
||||||
|
}
|
||||||
if len(c.AllowedWebmentionSources) == 0 {
|
if len(c.AllowedWebmentionSources) == 0 {
|
||||||
keys = append(keys, "allowedWebmentionSources")
|
keys = append(keys, "allowedWebmentionSources")
|
||||||
}
|
}
|
||||||
|
@ -97,7 +100,7 @@ func defaultConfig() *Config {
|
||||||
Token: "miauwkes",
|
Token: "miauwkes",
|
||||||
UtcOffset: 60,
|
UtcOffset: 60,
|
||||||
DataPath: "data",
|
DataPath: "data",
|
||||||
Connection: "data/mentions.db",
|
ConString: "data/mentions.db",
|
||||||
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"},
|
||||||
DisallowedWebmentionDomains: []string{"youtube.com"},
|
DisallowedWebmentionDomains: []string{"youtube.com"},
|
||||||
}
|
}
|
||||||
|
|
13
db/repo.go
13
db/repo.go
|
@ -13,8 +13,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type MentionRepoBunt struct {
|
type MentionRepoBunt struct {
|
||||||
db *buntdb.DB
|
db *buntdb.DB
|
||||||
conf *common.Config
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type MentionRepo interface {
|
type MentionRepo interface {
|
||||||
|
@ -79,7 +78,7 @@ func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, er
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string {
|
func (r *MentionRepoBunt) mentionToKey(wm mf.Mention) string {
|
||||||
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain(r.conf))
|
return fmt.Sprintf("%s:%s", wm.Key(), wm.Domain())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns a single unmarshalled json value based on the mention key.
|
// Get returns a single unmarshalled json value based on the mention key.
|
||||||
|
@ -129,12 +128,10 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult {
|
||||||
// It also creates necessary indexes based on the passed domain config.
|
// It also creates necessary indexes based on the passed domain config.
|
||||||
// This panics if it cannot open the db.
|
// This panics if it cannot open the db.
|
||||||
func NewMentionRepo(c *common.Config) *MentionRepoBunt {
|
func NewMentionRepo(c *common.Config) *MentionRepoBunt {
|
||||||
repo := &MentionRepoBunt{
|
repo := &MentionRepoBunt{}
|
||||||
conf: c,
|
db, err := buntdb.Open(c.ConString)
|
||||||
}
|
|
||||||
db, err := buntdb.Open(c.Connection)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Str("constr", c.Connection).Msg("new mention repo: cannot open db")
|
log.Fatal().Str("constr", c.ConString).Msg("new mention repo: cannot open db")
|
||||||
}
|
}
|
||||||
repo.db = db
|
repo.db = db
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@ import (
|
||||||
|
|
||||||
var (
|
var (
|
||||||
conf = &common.Config{
|
conf = &common.Config{
|
||||||
Connection: ":memory:",
|
ConString: ":memory:",
|
||||||
AllowedWebmentionSources: []string{
|
AllowedWebmentionSources: []string{
|
||||||
"pussycat.com",
|
"pussycat.com",
|
||||||
},
|
},
|
||||||
|
@ -23,7 +23,7 @@ var (
|
||||||
func TestDelete(t *testing.T) {
|
func TestDelete(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := NewMentionRepo(conf)
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
Target: "pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}
|
}
|
||||||
db.Save(wm, &mf.IndiewebData{
|
db.Save(wm, &mf.IndiewebData{
|
||||||
Name: "lolz",
|
Name: "lolz",
|
||||||
|
@ -56,7 +56,7 @@ func TestSinceFirstTimeIsEmptytime(t *testing.T) {
|
||||||
func TestGet(t *testing.T) {
|
func TestGet(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := NewMentionRepo(conf)
|
||||||
wm := mf.Mention{
|
wm := mf.Mention{
|
||||||
Target: "pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}
|
}
|
||||||
db.Save(wm, &mf.IndiewebData{
|
db.Save(wm, &mf.IndiewebData{
|
||||||
Name: "lolz",
|
Name: "lolz",
|
||||||
|
@ -69,7 +69,7 @@ func TestGet(t *testing.T) {
|
||||||
func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
||||||
defer os.Remove("test.db")
|
defer os.Remove("test.db")
|
||||||
db := NewMentionRepo(&common.Config{
|
db := NewMentionRepo(&common.Config{
|
||||||
Connection: "test.db",
|
ConString: "test.db",
|
||||||
AllowedWebmentionSources: []string{
|
AllowedWebmentionSources: []string{
|
||||||
"pussycat.com",
|
"pussycat.com",
|
||||||
},
|
},
|
||||||
|
@ -99,7 +99,7 @@ func BenchmarkMentionRepoBunt_GetAll(b *testing.B) {
|
||||||
func TestGetAllAndSaveSomeJson(t *testing.T) {
|
func TestGetAllAndSaveSomeJson(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := NewMentionRepo(conf)
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Target: "pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}, &mf.IndiewebData{
|
}, &mf.IndiewebData{
|
||||||
Name: "lolz",
|
Name: "lolz",
|
||||||
})
|
})
|
||||||
|
@ -112,12 +112,12 @@ func TestGetAllAndSaveSomeJson(t *testing.T) {
|
||||||
func TestGetFiltersBasedOnDomain(t *testing.T) {
|
func TestGetFiltersBasedOnDomain(t *testing.T) {
|
||||||
db := NewMentionRepo(conf)
|
db := NewMentionRepo(conf)
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Target: "pussycat.com/coolpussy.html",
|
Target: "https://pussycat.com/coolpussy.html",
|
||||||
}, &mf.IndiewebData{
|
}, &mf.IndiewebData{
|
||||||
Name: "lolz",
|
Name: "lolz",
|
||||||
})
|
})
|
||||||
db.Save(mf.Mention{
|
db.Save(mf.Mention{
|
||||||
Target: "dingeling.com/dogshateus.html",
|
Target: "https://dingeling.com/dogshateus.html",
|
||||||
}, &mf.IndiewebData{
|
}, &mf.IndiewebData{
|
||||||
Name: "amaigat",
|
Name: "amaigat",
|
||||||
})
|
})
|
||||||
|
|
|
@ -11,21 +11,24 @@ import (
|
||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
func mai() {
|
// Migrate migrates from data/[domain]/md5hash.json files to the new key/value db.
|
||||||
|
// This is only needed if you've run go-jamming before the db migration.
|
||||||
|
func Migrate() {
|
||||||
cnf := common.Configure()
|
cnf := common.Configure()
|
||||||
os.Remove(cnf.Connection)
|
dataPath := "data" // decoupled from config, change if needed
|
||||||
|
os.Remove(cnf.ConString)
|
||||||
repo := db.NewMentionRepo(cnf)
|
repo := db.NewMentionRepo(cnf)
|
||||||
|
|
||||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Starting migration...")
|
log.Info().Str("dbconfig", cnf.ConString).Msg("Starting migration...")
|
||||||
for _, domain := range cnf.AllowedWebmentionSources {
|
for _, domain := range cnf.AllowedWebmentionSources {
|
||||||
fmt.Printf("Processing domain %s...\n", domain)
|
fmt.Printf("Processing domain %s...\n", domain)
|
||||||
entries, err := os.ReadDir(fmt.Sprintf("%s/%s", cnf.DataPath, domain))
|
entries, err := os.ReadDir(fmt.Sprintf("%s/%s", dataPath, domain))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Err(err).Msg("Error while reading import path")
|
log.Fatal().Err(err).Msg("Error while reading import path")
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range entries {
|
for _, file := range entries {
|
||||||
filename := fmt.Sprintf("%s/%s/%s", cnf.DataPath, domain, file.Name())
|
filename := fmt.Sprintf("%s/%s/%s", dataPath, domain, file.Name())
|
||||||
data, err := ioutil.ReadFile(filename)
|
data, err := ioutil.ReadFile(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Str("file", filename).Err(err).Msg("Error while reading file")
|
log.Fatal().Str("file", filename).Err(err).Msg("Error while reading file")
|
||||||
|
@ -43,9 +46,9 @@ func mai() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Checking for since files...")
|
log.Info().Str("dbconfig", cnf.ConString).Msg("Checking for since files...")
|
||||||
for _, domain := range cnf.AllowedWebmentionSources {
|
for _, domain := range cnf.AllowedWebmentionSources {
|
||||||
since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", cnf.DataPath, domain))
|
since, err := ioutil.ReadFile(fmt.Sprintf("%s/%s-since.txt", dataPath, domain))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Str("domain", domain).Msg("No since found, skipping")
|
log.Warn().Str("domain", domain).Msg("No since found, skipping")
|
||||||
continue
|
continue
|
||||||
|
@ -55,5 +58,5 @@ func mai() {
|
||||||
repo.UpdateSince(domain, common.IsoToTime(string(since)))
|
repo.UpdateSince(domain, common.IsoToTime(string(since)))
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("dbconfig", cnf.Connection).Msg("Done! Check db")
|
log.Info().Str("dbconfig", cnf.ConString).Msg("Done! Check db")
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue