diff --git a/app/mf/mention.go b/app/mf/mention.go index 31f2b8e..03a8f98 100644 --- a/app/mf/mention.go +++ b/app/mf/mention.go @@ -1,10 +1,10 @@ package mf import ( + "brainbaking.com/go-jamming/rest" "crypto/md5" "fmt" "net/url" - "strings" ) // this should be passed along as a value object, not as a pointer @@ -28,12 +28,7 @@ func (wm Mention) String() string { // This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling. // Assumes http(s) protocol, which should have been validated by now. func (wm Mention) Domain() string { - withPossibleSubdomain := strings.Split(wm.Target, "/")[2] - split := strings.Split(withPossibleSubdomain, ".") - if len(split) == 2 { - return withPossibleSubdomain // that was the extention, not the subdomain. - } - return fmt.Sprintf("%s.%s", split[1], split[2]) + return rest.Domain(wm.Target) } func (wm Mention) Key() string { diff --git a/app/mf/mention_test.go b/app/mf/mention_test.go index 31367b4..a6b6be0 100644 --- a/app/mf/mention_test.go +++ b/app/mf/mention_test.go @@ -6,41 +6,10 @@ import ( ) func TestDomainParseFromTarget(t *testing.T) { - cases := []struct { - label string - target string - expected string - }{ - { - "parse from default http domain", - "http://patat.be/frietjes/zijn/lekker", - "patat.be", - }, - { - "parse from default https domain", - "https://frit.be/patatjes/zijn/lekker", - "frit.be", - }, - { - "parse from default https domain with www subdomain", - "https://www.frit.be/patatjes/zijn/lekker", - "frit.be", - }, - { - "parse from default https domain with some random subdomain", - "https://mayonaise.frit.be/patatjes/zijn/lekker", - "frit.be", - }, + wm := Mention{ + Source: "source", + Target: "http://patat.be/frietjes/zijn/lekker", } - for _, tc := range cases { - t.Run(tc.label, func(t *testing.T) { - wm := Mention{ - Source: "source", - Target: tc.target, - } - - assert.Equal(t, tc.expected, wm.Domain()) - }) - } + assert.Equal(t, "patat.be", wm.Domain()) } diff --git a/app/mf/microformats.go b/app/mf/microformats.go index 60794b3..7a420de 100644 --- a/app/mf/microformats.go +++ b/app/mf/microformats.go @@ -2,8 +2,6 @@ package mf import ( "brainbaking.com/go-jamming/common" - "encoding/json" - "io/ioutil" "strings" "time" "willnorris.com/go/microformats" @@ -41,17 +39,15 @@ type IndiewebData struct { Target string `json:"target"` } -func (id *IndiewebData) IsEmpty() bool { - return id.Url == "" +func (id *IndiewebData) AsMention() Mention { + return Mention{ + Source: id.Source, + Target: id.Target, + } } -// RequireFromFile converts the file JSON contents into the indieweb struct. -// This ignores read and marshall errors and returns an emtpy struct instead. -func RequireFromFile(file string) *IndiewebData { - indiewebData := &IndiewebData{} - data, _ := ioutil.ReadFile(file) - json.Unmarshal(data, indiewebData) - return indiewebData +func (id *IndiewebData) IsEmpty() bool { + return id.Url == "" } func PublishedNow(utcOffset int) string { diff --git a/app/pictures/handler.go b/app/pictures/handler.go new file mode 100644 index 0000000..1873d09 --- /dev/null +++ b/app/pictures/handler.go @@ -0,0 +1,23 @@ +package pictures + +import ( + "brainbaking.com/go-jamming/db" + "github.com/gorilla/mux" + "net/http" +) + +// Handle handles picture GET calls. +// It does not validate the picture query as it's part of a composite key anyway. +func Handle(repo db.MentionRepo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + picDomain := mux.Vars(r)["picture"] + picData := repo.GetPicture(picDomain) + if picData == nil { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + // TODO response headers? is this a jpeg, png, gif, webm? should we? + w.Write(picData) + } +} diff --git a/app/routes.go b/app/routes.go index eab0112..21984e5 100644 --- a/app/routes.go +++ b/app/routes.go @@ -2,6 +2,7 @@ package app import ( "brainbaking.com/go-jamming/app/index" + "brainbaking.com/go-jamming/app/pictures" "brainbaking.com/go-jamming/app/pingback" "brainbaking.com/go-jamming/app/webmention" ) @@ -14,6 +15,7 @@ func (s *server) routes() { db := s.repo s.router.HandleFunc("/", index.Handle(c)).Methods("GET") + s.router.HandleFunc("/pictures/{picture}", pictures.Handle(db)).Methods("GET") s.router.HandleFunc("/pingback", pingback.HandlePost(c, db)).Methods("POST") s.router.HandleFunc("/webmention", webmention.HandlePost(c, db)).Methods("POST") s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleGet(db))).Methods("GET") diff --git a/app/webmention/recv/receive.go b/app/webmention/recv/receive.go index 77f182f..df6675c 100644 --- a/app/webmention/recv/receive.go +++ b/app/webmention/recv/receive.go @@ -5,6 +5,7 @@ import ( "brainbaking.com/go-jamming/common" "brainbaking.com/go-jamming/db" "brainbaking.com/go-jamming/rest" + "fmt" "regexp" "strings" @@ -41,6 +42,9 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) { data := microformats.Parse(strings.NewReader(body), wm.SourceUrl()) indieweb := recv.convertBodyToIndiewebData(body, wm, mf.HEntry(data)) + if indieweb.Author.Picture != "" { + recv.saveAuthorPictureLocally(indieweb) + } key, err := recv.Repo.Save(wm, indieweb) if err != nil { @@ -93,6 +97,22 @@ func (recv *Receiver) parseBodyAsNonIndiewebSite(body string, wm mf.Mention) *mf } } +func (recv *Receiver) saveAuthorPictureLocally(indieweb *mf.IndiewebData) { + _, picData, err := recv.RestClient.GetBody(indieweb.Author.Picture) + if err != nil { + log.Warn().Err(err).Str("url", indieweb.Author.Picture).Msg("Unable to download author picture. Ignoring.") + return + } + srcDomain := rest.Domain(indieweb.Source) + _, dberr := recv.Repo.SavePicture(picData, srcDomain) + if dberr != nil { + log.Warn().Err(err).Str("url", indieweb.Author.Picture).Msg("Unable to save downloaded author picture. Ignoring.") + return + } + + indieweb.Author.Picture = fmt.Sprintf("/pictures/%s", srcDomain) +} + func nonIndiewebTitle(body string, wm mf.Mention) string { titleMatch := titleRegexp.FindStringSubmatch(body) title := wm.Source diff --git a/app/webmention/recv/receive_test.go b/app/webmention/recv/receive_test.go index 4490561..1814838 100644 --- a/app/webmention/recv/receive_test.go +++ b/app/webmention/recv/receive_test.go @@ -34,7 +34,7 @@ func TestReceive(t *testing.T) { Source: "https://brainbaking.com/valid-bridgy-twitter-source.html", Target: "https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag", }, - json: `{"author":{"name":"Jamie Tanna","picture":"https://www.jvt.me/img/profile.png"},"name":"","content":"Recommended read:\nThe IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions\nhttps://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/","published":"2021-03-15T12:42:00+0000","url":"https://brainbaking.com/mf2/2021/03/1bkre/","type":"bookmark","source":"https://brainbaking.com/valid-bridgy-twitter-source.html","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag"}`, + json: `{"author":{"name":"Jamie Tanna","picture":"/pictures/brainbaking.com"},"name":"","content":"Recommended read:\nThe IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions\nhttps://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/","published":"2021-03-15T12:42:00+0000","url":"https://brainbaking.com/mf2/2021/03/1bkre/","type":"bookmark","source":"https://brainbaking.com/valid-bridgy-twitter-source.html","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag"}`, }, { label: "receive a brid.gy Webmention like", @@ -44,7 +44,7 @@ func TestReceive(t *testing.T) { Target: "https://brainbaking.com/valid-indieweb-target.html", }, // no dates in bridgy-to-mastodon likes... - json: `{"author":{"name":"Stampeding Longhorn","picture":"https://cdn.social.linux.pizza/v1/AUTH_91eb37814936490c95da7b85993cc2ff/sociallinuxpizza/accounts/avatars/000/185/996/original/9e36da0c093cfc9b.png"},"name":"","content":"","published":"2020-01-01T12:30:00","url":"https://chat.brainbaking.com/notice/A4nx1rFwKUJYSe4TqK#favorited-by-A4nwg4LYyh4WgrJOXg","type":"like","source":"https://brainbaking.com/valid-bridgy-like.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`, + json: `{"author":{"name":"Stampeding Longhorn","picture":"/pictures/brainbaking.com"},"name":"","content":"","published":"2020-01-01T12:30:00","url":"https://chat.brainbaking.com/notice/A4nx1rFwKUJYSe4TqK#favorited-by-A4nwg4LYyh4WgrJOXg","type":"like","source":"https://brainbaking.com/valid-bridgy-like.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`, }, { label: "receive a brid.gy Webmention that has a url and photo without value", @@ -52,7 +52,7 @@ func TestReceive(t *testing.T) { Source: "https://brainbaking.com/valid-bridgy-source.html", Target: "https://brainbaking.com/valid-indieweb-target.html", }, - json: `{"author":{"name":"Stampeding Longhorn", "picture":"https://cdn.social.linux.pizza/v1/AUTH_91eb37814936490c95da7b85993cc2ff/sociallinuxpizza/accounts/avatars/000/185/996/original/9e36da0c093cfc9b.png"}, "content":"@wouter The cat pictures are awesome. for jest tests!", "name":"@wouter The cat pictures are awesome. for jest tests!", "published":"2021-03-02T16:17:18.000Z", "source":"https://brainbaking.com/valid-bridgy-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://social.linux.pizza/@StampedingLonghorn/105821099684887793"}`, + json: `{"author":{"name":"Stampeding Longhorn", "picture":"/pictures/brainbaking.com"}, "content":"@wouter The cat pictures are awesome. for jest tests!", "name":"@wouter The cat pictures are awesome. for jest tests!", "published":"2021-03-02T16:17:18.000Z", "source":"https://brainbaking.com/valid-bridgy-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://social.linux.pizza/@StampedingLonghorn/105821099684887793"}`, }, { label: "receive saves a JSON file of indieweb-metadata if all is valid", @@ -60,7 +60,7 @@ func TestReceive(t *testing.T) { Source: "https://brainbaking.com/valid-indieweb-source.html", Target: "https://jefklakscodex.com/articles", }, - json: `{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com//img/avatar.jpg"},"name":"I just learned about https://www.inklestudios.com/...","content":"This is cool, I just found out about valid indieweb target - so cool","published":"2021-03-06T12:41:00","url":"https://brainbaking.com/notes/2021/03/06h12m41s48/","type":"mention","source":"https://brainbaking.com/valid-indieweb-source.html","target":"https://jefklakscodex.com/articles"}`, + json: `{"author":{"name":"Wouter Groeneveld","picture":"/pictures/brainbaking.com"},"name":"I just learned about https://www.inklestudios.com/...","content":"This is cool, I just found out about valid indieweb target - so cool","published":"2021-03-06T12:41:00","url":"https://brainbaking.com/notes/2021/03/06h12m41s48/","type":"mention","source":"https://brainbaking.com/valid-indieweb-source.html","target":"https://jefklakscodex.com/articles"}`, }, { label: "receive saves a JSON file of indieweb-metadata with summary as content if present", @@ -68,7 +68,7 @@ func TestReceive(t *testing.T) { Source: "https://brainbaking.com/valid-indieweb-source-with-summary.html", Target: "https://brainbaking.com/valid-indieweb-target.html", }, - json: `{"author":{"name":"Wouter Groeneveld", "picture":"https://brainbaking.com//img/avatar.jpg"}, "content":"This is cool, this is a summary!", "name":"I just learned about https://www.inklestudios.com/...", "published":"2021-03-06T12:41:00", "source":"https://brainbaking.com/valid-indieweb-source-with-summary.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/notes/2021/03/06h12m41s48/"}`, + json: `{"author":{"name":"Wouter Groeneveld", "picture":"/pictures/brainbaking.com"}, "content":"This is cool, this is a summary!", "name":"I just learned about https://www.inklestudios.com/...", "published":"2021-03-06T12:41:00", "source":"https://brainbaking.com/valid-indieweb-source-with-summary.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/notes/2021/03/06h12m41s48/"}`, }, { label: "receive saves a JSON file of non-indieweb-data such as title if all is valid", diff --git a/common/config.go b/common/config.go index c6b039f..f44a233 100644 --- a/common/config.go +++ b/common/config.go @@ -26,9 +26,6 @@ func (c *Config) missingKeys() []string { if c.Token == "" { keys = append(keys, "token") } - if c.DataPath == "" { - keys = append(keys, "dataPath") - } if c.ConString == "" { keys = append(keys, "conString") } @@ -66,11 +63,11 @@ func (c *Config) FetchDomain(url string) (string, error) { } func Configure() *Config { - conf := config() - for _, domain := range conf.AllowedWebmentionSources { + c := config() + for _, domain := range c.AllowedWebmentionSources { log.Info().Str("allowedDomain", domain).Msg("Configured") } - return conf + return c } func config() *Config { @@ -99,7 +96,6 @@ func defaultConfig() *Config { Port: 1337, Token: "miauwkes", UtcOffset: 60, - DataPath: "data", ConString: "data/mentions.db", AllowedWebmentionSources: []string{"brainbaking.com", "jefklakscodex.com"}, DisallowedWebmentionDomains: []string{"youtube.com"}, diff --git a/migrate-db.go b/db/migrate-db.go similarity index 66% rename from migrate-db.go rename to db/migrate-db.go index 8b005b6..dfe3875 100644 --- a/migrate-db.go +++ b/db/migrate-db.go @@ -1,9 +1,8 @@ -package main +package db import ( "brainbaking.com/go-jamming/app/mf" "brainbaking.com/go-jamming/common" - "brainbaking.com/go-jamming/db" "encoding/json" "fmt" "github.com/rs/zerolog/log" @@ -11,20 +10,19 @@ import ( "os" ) -// Migrate migrates from data/[domain]/md5hash.json files to the new key/value db. -// This is only needed if you've run go-jamming before the db migration. -func Migrate() { - cnf := common.Configure() - dataPath := "data" // decoupled from config, change if needed - os.Remove(cnf.ConString) - repo := db.NewMentionRepo(cnf) +const ( + dataPath = "data" // decoupled from config, change if needed +) - log.Info().Str("dbconfig", cnf.ConString).Msg("Starting migration...") +// MigrateDataFiles migrates from data/[domain]/md5hash.json files to the new key/value db. +// This is only needed if you've run go-jamming before the db migration. +func MigrateDataFiles(cnf *common.Config, repo *MentionRepoBunt) { for _, domain := range cnf.AllowedWebmentionSources { - fmt.Printf("Processing domain %s...\n", domain) + log.Info().Str("domain", domain).Msg("MigrateDataFiles: processing") entries, err := os.ReadDir(fmt.Sprintf("%s/%s", dataPath, domain)) if err != nil { - log.Fatal().Err(err).Msg("Error while reading import path") + log.Warn().Err(err).Msg("Error while reading import path - migration could be already done...") + continue } for _, file := range entries { @@ -36,10 +34,7 @@ func Migrate() { var indiewebData mf.IndiewebData json.Unmarshal(data, &indiewebData) - mention := mf.Mention{ - Source: indiewebData.Source, - Target: indiewebData.Target, - } + mention := indiewebData.AsMention() log.Info().Stringer("wm", mention).Str("file", filename).Msg("Re-saving entry") repo.Save(mention, &indiewebData) @@ -57,6 +52,4 @@ func Migrate() { log.Info().Str("domain", domain).Str("since", string(since)).Msg("Saving since") repo.UpdateSince(domain, common.IsoToTime(string(since))) } - - log.Info().Str("dbconfig", cnf.ConString).Msg("Done! Check db") } diff --git a/db/migrate-pictures.go b/db/migrate-pictures.go new file mode 100644 index 0000000..b1faa38 --- /dev/null +++ b/db/migrate-pictures.go @@ -0,0 +1,69 @@ +package db + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "brainbaking.com/go-jamming/rest" + "fmt" + "github.com/rs/zerolog/log" + "strings" +) + +// MigratePictures converts all indiewebdata already present in the database into local byte arrays (strings). +// This makes it possible to self-host author pictures. Run only after Migrate() in migrate-db.go. +func MigratePictures(cnf *common.Config, repo *MentionRepoBunt) { + for _, domain := range cnf.AllowedWebmentionSources { + all := repo.GetAll(domain) + log.Info().Str("domain", domain).Int("mentions", len(all.Data)).Msg("migrate pictures: processing") + for _, mention := range all.Data { + if mention.Author.Picture == "" { + log.Warn().Str("url", mention.Url).Msg("Mention without author picture, skipping") + continue + } + + savePicture(mention, repo, cnf) + } + } +} + +// ChangeBaseUrl changes all base urls of pictures in the database. +// e.g. "http://localhost:1337/" to "https://jam.brainbaking.com/" +func ChangeBaseUrl(old, new string) { + cnf := common.Configure() + repo := NewMentionRepo(cnf) + + for _, domain := range cnf.AllowedWebmentionSources { + for _, mention := range repo.GetAll(domain).Data { + if mention.Author.Picture == "" { + log.Warn().Str("url", mention.Url).Msg("Mention without author picture, skipping") + continue + } + mention.Author.Picture = strings.ReplaceAll(mention.Author.Picture, old, new) + repo.Save(mention.AsMention(), mention) + } + } +} + +func savePicture(indieweb *mf.IndiewebData, repo *MentionRepoBunt, cnf *common.Config) { + restClient := &rest.HttpClient{} + picUrl := indieweb.Author.Picture + log.Info().Str("oldurl", picUrl).Msg("About to cache picture") + _, picData, err := restClient.GetBody(picUrl) + if err != nil { + log.Warn().Err(err).Str("url", picUrl).Msg("Unable to download author picture. Ignoring.") + return + } + srcDomain := rest.Domain(indieweb.Source) + _, dberr := repo.SavePicture(picData, srcDomain) + if dberr != nil { + log.Warn().Err(err).Str("url", picUrl).Msg("Unable to save downloaded author picture. Ignoring.") + return + } + + indieweb.Author.Picture = fmt.Sprintf("/pictures/%s", srcDomain) + _, serr := repo.Save(indieweb.AsMention(), indieweb) + if serr != nil { + log.Fatal().Err(serr).Msg("Unable to update wm?") + } + log.Info().Str("oldurl", picUrl).Str("newurl", indieweb.Author.Picture).Msg("Picture saved!") +} diff --git a/db/migrate.go b/db/migrate.go new file mode 100644 index 0000000..f65b287 --- /dev/null +++ b/db/migrate.go @@ -0,0 +1,13 @@ +package db + +import "brainbaking.com/go-jamming/common" + +// Migrate self-checks and executes necessary DB migrations, if any. +func Migrate() { + cnf := common.Configure() + repo := NewMentionRepo(cnf) + + MigrateDataFiles(cnf, repo) + MigratePictures(cnf, repo) + repo.db.Shrink() +} diff --git a/db/repo.go b/db/repo.go index 4d9436f..06affd3 100644 --- a/db/repo.go +++ b/db/repo.go @@ -18,17 +18,19 @@ type MentionRepoBunt struct { type MentionRepo interface { Save(key mf.Mention, data *mf.IndiewebData) (string, error) + SavePicture(bytes string, domain string) (string, error) Delete(key mf.Mention) Since(domain string) (time.Time, error) UpdateSince(domain string, since time.Time) Get(key mf.Mention) *mf.IndiewebData + GetPicture(domain string) []byte GetAll(domain string) mf.IndiewebDataResult } // UpdateSince updates the since timestamp to now. Logs but ignores errors. func (r *MentionRepoBunt) UpdateSince(domain string, since time.Time) { err := r.db.Update(func(tx *buntdb.Tx) error { - _, _, err := tx.Set(fmt.Sprintf("%s:since", domain), common.TimeToIso(since), nil) + _, _, err := tx.Set(sinceKey(domain), common.TimeToIso(since), nil) return err }) if err != nil { @@ -41,7 +43,7 @@ func (r *MentionRepoBunt) UpdateSince(domain string, since time.Time) { func (r *MentionRepoBunt) Since(domain string) (time.Time, error) { var since string err := r.db.View(func(tx *buntdb.Tx) error { - val, err := tx.Get(fmt.Sprintf("%s:since", domain)) + val, err := tx.Get(sinceKey(domain)) since = val return err }) @@ -51,6 +53,10 @@ func (r *MentionRepoBunt) Since(domain string) (time.Time, error) { return common.IsoToTime(since), nil } +func sinceKey(domain string) string { + return fmt.Sprintf("%s:since", domain) +} + // Delete removes a possibly present mention by key. Ignores possible errors. func (r *MentionRepoBunt) Delete(wm mf.Mention) { key := r.mentionToKey(wm) @@ -60,6 +66,22 @@ func (r *MentionRepoBunt) Delete(wm mf.Mention) { }) } +func (r *MentionRepoBunt) SavePicture(bytes string, domain string) (string, error) { + key := pictureKey(domain) + err := r.db.Update(func(tx *buntdb.Tx) error { + _, _, err := tx.Set(key, bytes, nil) + return err + }) + if err != nil { + return "", err + } + return key, nil +} + +func pictureKey(domain string) string { + return fmt.Sprintf("%s:picture", domain) +} + // Save saves the mention by marshalling data. Returns the key or a marshal/persist error. func (r *MentionRepoBunt) Save(wm mf.Mention, data *mf.IndiewebData) (string, error) { jsonData, err := json.Marshal(data) @@ -104,6 +126,24 @@ func (r *MentionRepoBunt) Get(wm mf.Mention) *mf.IndiewebData { return &data } +func (r *MentionRepoBunt) GetPicture(domain string) []byte { + var data []byte + key := pictureKey(domain) + err := r.db.View(func(tx *buntdb.Tx) error { + val, err := tx.Get(key) + if err != nil { + return err + } + data = []byte(val) + return nil + }) + if err != nil { + log.Error().Err(err).Str("key", key).Msg("repo getpicture: unable to retrieve key") + return nil + } + return data +} + // GetAll returns a wrapped data result for all mentions for a particular domain. // Intentionally ignores marshal errors, db should be consistent! // Warning, this will potentially marshall 10k strings! @@ -117,6 +157,7 @@ func (r *MentionRepoBunt) GetAll(domain string) mf.IndiewebDataResult { return true }) }) + if err != nil { log.Error().Err(err).Msg("get all: failed to ascend from view") return mf.IndiewebDataResult{} diff --git a/db/repo_test.go b/db/repo_test.go index c963835..e6095f7 100644 --- a/db/repo_test.go +++ b/db/repo_test.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/stretchr/testify/assert" "github.com/tidwall/buntdb" + "io/ioutil" "os" "testing" "time" @@ -20,6 +21,19 @@ var ( } ) +func TestSaveAndGetPicture(t *testing.T) { + data, err := ioutil.ReadFile("../mocks/picture.jpg") + assert.NoError(t, err) + + db := NewMentionRepo(conf) + key, dberr := db.SavePicture(string(data), "bloeberig.be") + assert.NoError(t, dberr) + assert.Equal(t, "bloeberig.be:picture", key) + + picDataAfterSave := db.GetPicture("bloeberig.be") + assert.Equal(t, data, picDataAfterSave) +} + func TestDelete(t *testing.T) { db := NewMentionRepo(conf) wm := mf.Mention{ diff --git a/go.mod b/go.mod index 0dda4fc..dddf83c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.8 github.com/rs/zerolog v1.21.0 github.com/stretchr/testify v1.7.0 - github.com/tidwall/buntdb v1.2.3 // indirect + github.com/tidwall/buntdb v1.2.3 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba willnorris.com/go/microformats v1.1.1 ) diff --git a/go.sum b/go.sum index e4b1d47..7c14741 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8= github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY= github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ= +github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= diff --git a/main.go b/main.go index 04a95b1..7af916c 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "brainbaking.com/go-jamming/db" "flag" "os" @@ -14,15 +15,27 @@ func main() { zerolog.TimeFieldFormat = zerolog.TimeFormatUnix verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)") + migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.") flag.Parse() // logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack. zerolog.SetGlobalLevel(zerolog.InfoLevel) - if *verboseFlag == true { + if *verboseFlag == true || *migrateFlag == true { log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) zerolog.SetGlobalLevel(zerolog.DebugLevel) } + if *migrateFlag == true { + migrate() + os.Exit(0) + } + log.Debug().Msg("Let's a go!") app.Start() } + +func migrate() { + log.Info().Msg("Starting db migration...") + db.Migrate() + log.Info().Msg("Migration ended, exiting.") +} diff --git a/mentions.db b/mentions.db new file mode 100644 index 0000000..544219c --- /dev/null +++ b/mentions.db @@ -0,0 +1,378 @@ +*3 +$3 +set +$48 +18acebf759df6d79f8a109faa8d18fc7:brainbaking.com +$625 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I changed my mind on my toot syndication to brainb...","content":"I changed my mind on my toot syndication to brainbaking.com policy. From now on, only non-replies (in-reply-to) get pushed to https://brainbaking.com/notes/ - it was cluttering up the RSS feed and most replies are useless to non-followers anyway. Sor...","published":"2021-03-20T13:27:00","url":"https://brainbaking.com/notes/2021/03/20h13m27s36/","type":"mention","source":"https://brainbaking.com/notes/2021/03/20h13m27s36/","target":"http://brainbaking.com"} +*3 +$3 +set +$48 +1b2ba8c60768014afa147580d452ced5:brainbaking.com +$633 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I changed my mind on my toot syndication to brainb...","content":"I changed my mind on my toot syndication to brainbaking.com policy. From now on, only non-replies (in-reply-to) get pushed to https://brainbaking.com/notes/ - it was cluttering up the RSS feed and most replies are useless to non-followers anyway. Sor...","published":"2021-03-20T13:27:00","url":"https://brainbaking.com/notes/2021/03/20h13m27s36/","type":"mention","source":"https://brainbaking.com/notes/2021/03/20h13m27s36/","target":"https://brainbaking.com/notes/"} +*3 +$3 +set +$48 +421c001c7efa6e36e6f331c005d31c27:brainbaking.com +$597 +{"author":{"name":"Henrique Dias","picture":"https://hacdias.com/me-256.jpg"},"name":"Site Ideas","content":"A bunch of ideas for my website that might never get implemented.Wanna know more about the website? Check out the meta page!Finish uses.https://cleanuptheweb.org/Create /meta/historical with pictures of old versions of my website.Remove .Params.socia...","published":"2021-04-17T06:46:33Z","url":"https://hacdias.com/notes/site-ideas/","type":"mention","source":"https://hacdias.com/notes/site-ideas","target":"https://brainbaking.com/post/2021/04/using-hugo-to-launch-a-gemini-capsule/"} +*3 +$3 +set +$48 +4b0d1434331f1a382493b2838dc4b1a2:brainbaking.com +$421 +{"author":{"name":"Jamie Tanna","picture":"https://www.jvt.me/img/profile.png"},"name":"","content":"Recommended read: The IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions","published":"2021-03-15T12:42:00+0000","url":"https://www.jvt.me/mf2/2021/03/1bkre/","type":"mention","source":"https://www.jvt.me/mf2/2021/03/1bkre/","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/"} +*3 +$3 +set +$48 +5854ce85363d8c7513cd14e52870d46b:brainbaking.com +$708 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"@StampedingLonghorn @256 Don't forget the cleverly hidden Roland MT-32, a majestic piece of p...","content":"@StampedingLonghorn @256 Don't forget the cleverly hidden Roland MT-32, a majestic piece of pre-MIDI standardized era synthesizer. What else would you use to run Sierra Online games, and monkey1? I really need one for my 486… https://brainbaking.com/...","published":"2021-03-02T17:13:00","url":"https://brainbaking.com/notes/2021/03/02h17m13s27/","type":"mention","source":"https://brainbaking.com/notes/2021/03/02h17m13s27/","target":"https://brainbaking.com/post/2021/02/my-retro-desktop-setup/"} +*3 +$3 +set +$48 +69c99dcf12c180b8acb3c1c218d1f388:brainbaking.com +$655 +{"author":{"name":"Jefklak","picture":"https://jefklakscodex.com/img/avatar.jpg"},"name":"Reviews from 2001 revived!","content":"Good news everyone! Futurama might not be back, but old PC gaming previews and reviews from UnionVault.NET, one of Jefklak’s Codex ancestors, have been revived. I happened to stumble upon a few readable HTML files wile looking for something else and ...","published":"2020-09-25","url":"https://jefklakscodex.com/articles/features/reviews-from-2001-revived/","type":"mention","source":"https://jefklakscodex.com/articles/features/reviews-from-2001-revived/","target":"https://brainbaking.com/post/2020/09/reviving-a-80486/"} +*3 +$3 +set +$48 +78e327d69a3f5fa151f660b5a25c06a9:brainbaking.com +$682 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Heads up RSS feed readers of brainbaking.com! Federated half-baked thoughts (https://brainbaking....","content":"Heads up RSS feed readers of brainbaking.com! Federated half-baked thoughts (https://brainbaking.com/notes/) are now integrated in /index.xml 🤓. Don't like that? Subscribe to /post/index.xml instead! Next up: webmentions, PESOS-ing of Goodreads revi...","published":"2021-03-03T16:00:00","url":"https://brainbaking.com/notes/2021/03/03h16m00s44/","type":"mention","source":"https://brainbaking.com/notes/2021/03/03h16m00s44/","target":"https://brainbaking.com/notes/"} +*3 +$3 +set +$48 +792ed44939d1396d5156f1589e95b786:brainbaking.com +$707 +{"author":{"name":"Jefklak","picture":"https://jefklakscodex.com/img/avatar.jpg"},"name":"Rainbow Six 3: Raven Shield - 17 Years Later","content":"It’s amazing that the second disk is still readable by my Retro WinXP machine. It has been heavily abused in 2003 and the years after that. Rainbow Six' third installment, Raven Shield (or simply RvS), is quite a departure from the crude looking Rogu...","published":"2020-11-01","url":"https://jefklakscodex.com/articles/retrospectives/raven-shield-17-years-later/","type":"mention","source":"https://jefklakscodex.com/articles/retrospectives/raven-shield-17-years-later/","target":"https://brainbaking.com/post/2020/10/building-a-core2duo-winxp-retro-pc/"} +*3 +$3 +set +$48 +83313bc67b0730459876ab7c3b9738f5:brainbaking.com +$674 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Heads up RSS feed readers of brainbaking.com! Federated half-baked thoughts (https://brainbaking....","content":"Heads up RSS feed readers of brainbaking.com! Federated half-baked thoughts (https://brainbaking.com/notes/) are now integrated in /index.xml 🤓. Don't like that? Subscribe to /post/index.xml instead! Next up: webmentions, PESOS-ing of Goodreads revi...","published":"2021-03-03T16:00:00","url":"https://brainbaking.com/notes/2021/03/03h16m00s44/","type":"mention","source":"https://brainbaking.com/notes/2021/03/03h16m00s44/","target":"http://brainbaking.com"} +*3 +$3 +set +$48 +9427d6a53258e670b0988bbd7f9a6ea8:brainbaking.com +$704 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I've been fiddling with IndieWeb stuff the last week and all in all, I think it's a mixed...","content":"I've been fiddling with IndieWeb stuff the last week and all in all, I think it's a mixed bag: https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/@kev after I published it, I found out your \"removing support for indieweb\" post. Seems like we...","published":"2021-03-09T15:17:00","url":"https://brainbaking.com/notes/2021/03/09h15m17s30/","type":"mention","source":"https://brainbaking.com/notes/2021/03/09h15m17s30/","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/"} +*3 +$3 +set +$48 +d6a0a260f50553c15cd4f7833694c841:brainbaking.com +$727 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-ri...","content":"I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ Very impressed so far! Also rely on my own CalDAV server to replace GCalendar. Any others here running #lineageos for priv...","published":"2021-03-01T20:03:00","url":"https://brainbaking.com/notes/2021/03/01h20m03s35/","type":"mention","source":"https://brainbaking.com/notes/2021/03/01h20m03s35/","target":"https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/"} +*3 +$3 +set +$48 +f101a8437084c39583a39acbcbd569c5:brainbaking.com +$581 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"@rsolva that's a lie indeed 😁 see https://brainba...","content":"@rsolva that’s a lie indeed 😁 see https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ I use davx5 and it works flawlessly","published":"2021-03-13T09:58:00","url":"https://brainbaking.com/notes/2021/03/13h09m58s25/","type":"mention","source":"https://brainbaking.com/notes/2021/03/13h09m58s25/","target":"https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/"} +*3 +$3 +set +$50 +0f35e81e7376278498b425df5eaf956e:jefklakscodex.com +$610 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\n Enclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"} +*3 +$3 +set +$50 +169d68f8372aca44b39a3074a80429d8:jefklakscodex.com +$487 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"The insanity of collecting retro games","content":"Is a physical collection really worth it?","published":"2021-02-21","url":"https://brainbaking.com/post/2021/02/the-insanity-of-retro-game-collecting/","type":"mention","source":"https://brainbaking.com/post/2021/02/the-insanity-of-retro-game-collecting/","target":"https://jefklakscodex.com/articles/features/super-mario-64-aged-badly/"} +*3 +$3 +set +$50 +3bee3dedf14a37425fc4d3ec2056b135:jefklakscodex.com +$447 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Building an Athlon Windows 98 Retro PC","content":"Gaming from Quake to Quake III: Arena!","published":"2020-10-17","url":"https://brainbaking.com/post/2020/10/building-an-athlon-win98-retro-pc/","type":"mention","source":"https://brainbaking.com/post/2020/10/building-an-athlon-win98-retro-pc/","target":"https://jefklakscodex.com/tags/wizardry8/"} +*3 +$3 +set +$50 +460e6c81d63550d9bfb79051bb86eb11:jefklakscodex.com +$489 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"The insanity of collecting retro games","content":"Is a physical collection really worth it?","published":"2021-02-21","url":"https://brainbaking.com/post/2021/02/the-insanity-of-retro-game-collecting/","type":"mention","source":"https://brainbaking.com/post/2021/02/the-insanity-of-retro-game-collecting/","target":"https://jefklakscodex.com/articles/features/gaming-setup-2007-flashback/"} +*3 +$3 +set +$50 +509cf066810eadacb400e2397de5ed86:jefklakscodex.com +$473 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Win98 Upgrade: GeForce 3 Ti200 vs Riva TNT2","content":"Get more out of that AGPx4 slot!","published":"2021-01-28","url":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","type":"mention","source":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","target":"https://jefklakscodex.com/articles/features/the-best-and-worst-retro-hack-and-slash-games/"} +*3 +$3 +set +$50 +6880bf0f22930290ace839175db2ea34:jefklakscodex.com +$424 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Win98 Upgrade: GeForce 3 Ti200 vs Riva TNT2","content":"Get more out of that AGPx4 slot!","published":"2021-01-28","url":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","type":"mention","source":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","target":"https://jefklakscodex.com/tags/wizardry8/"} +*3 +$3 +set +$50 +6c97b9d5bf75e6601d3560dc3ef43771:jefklakscodex.com +$482 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"The Internet Killed Secrets in Games","content":"What's the 'secret' of the secret cow level in Diablo II?","published":"2020-11-19","url":"https://brainbaking.com/post/2020/11/the-internet-killed-secrets-in-games/","type":"mention","source":"https://brainbaking.com/post/2020/11/the-internet-killed-secrets-in-games/","target":"https://jefklakscodex.com/articles/reviews/gobliins2/"} +*3 +$3 +set +$50 +7ea008404c0004e592dc84213253d942:jefklakscodex.com +$479 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"The Internet Killed Secrets in Games","content":"What's the 'secret' of the secret cow level in Diablo II?","published":"2020-11-19","url":"https://brainbaking.com/post/2020/11/the-internet-killed-secrets-in-games/","type":"mention","source":"https://brainbaking.com/post/2020/11/the-internet-killed-secrets-in-games/","target":"https://jefklakscodex.com/articles/reviews/sacred/"} +*3 +$3 +set +$50 +8fc122b91788ceccb834ae721031ad98:jefklakscodex.com +$440 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"Win98 Upgrade: GeForce 3 Ti200 vs Riva TNT2","content":"Get more out of that AGPx4 slot!","published":"2021-01-28","url":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","type":"mention","source":"https://brainbaking.com/post/2021/01/win98-upgrade-geforce3/","target":"https://jefklakscodex.com/articles/reviews/dungeon-siege/"} +*3 +$3 +set +$50 +e8cd7a857456a9de776e7a13c982516c:jefklakscodex.com +$502 +{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"A journey through the history of webdesign","content":"Using personal websites and the Internet Archive","published":"2020-10-04","url":"https://brainbaking.com/post/2020/10/a-personal-journey-through-the-history-of-webdesign/","type":"mention","source":"https://brainbaking.com/post/2020/10/a-personal-journey-through-the-history-of-webdesign/","target":"https://jefklakscodex.com/tags/baldurs-gate-2/"} +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T12:54:38.452Z +*3 +$3 +set +$23 +jefklakscodex.com:since +$24 +2021-04-16T14:40:36.133Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T13:36:36.230Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T13:37:44.188Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T13:38:26.658Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T13:49:17.147Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T14:47:47.351Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T15:51:26.080Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T17:01:58.204Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T17:55:20.524Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T18:53:36.453Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T19:46:21.245Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T20:47:47.622Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T21:49:08.986Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T22:51:16.375Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-18T23:50:29.318Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T01:07:35.145Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T02:08:12.030Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T02:56:24.545Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T03:52:37.159Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T04:53:16.053Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T05:50:13.478Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T06:53:12.051Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T07:48:50.675Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T08:50:56.390Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T09:50:59.391Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T10:50:27.567Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T11:47:46.936Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T12:55:55.440Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T13:50:11.048Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T14:48:35.217Z +*3 +$3 +set +$21 +brainbaking.com:since +$24 +2021-04-19T15:51:50.183Z diff --git a/mocks/picture.jpg b/mocks/picture.jpg new file mode 100644 index 0000000..7151c28 Binary files /dev/null and b/mocks/picture.jpg differ diff --git a/mocks/valid-bridgy-like.html b/mocks/valid-bridgy-like.html index 6a5389f..ce8d228 100644 --- a/mocks/valid-bridgy-like.html +++ b/mocks/valid-bridgy-like.html @@ -32,7 +32,7 @@ body { Stampeding Longhorn StampedingLonghorn - + diff --git a/mocks/valid-bridgy-source.html b/mocks/valid-bridgy-source.html index fe19e0d..d4b1c00 100644 --- a/mocks/valid-bridgy-source.html +++ b/mocks/valid-bridgy-source.html @@ -32,7 +32,7 @@ body { Stampeding Longhorn StampedingLonghorn - + https://social.linux.pizza/@StampedingLonghorn/105821099684887793 diff --git a/mocks/valid-bridgy-twitter-source.html b/mocks/valid-bridgy-twitter-source.html index 0946ff3..454cdfc 100644 --- a/mocks/valid-bridgy-twitter-source.html +++ b/mocks/valid-bridgy-twitter-source.html @@ -1,6 +1,6 @@ The IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions · Jamie Tanna | Software Engineer

The IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions

Recommended read: The IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions -https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/

by Jamie Tanna's profile image Jamie Tanna +https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/

by Jamie Tanna's profile image Jamie Tanna . #indieweb #decentralisation diff --git a/mocks/valid-indieweb-source-with-summary.html b/mocks/valid-indieweb-source-with-summary.html index 300b239..357c8e6 100644 --- a/mocks/valid-indieweb-source-with-summary.html +++ b/mocks/valid-indieweb-source-with-summary.html @@ -275,7 +275,7 @@

diff --git a/mocks/valid-indieweb-source.html b/mocks/valid-indieweb-source.html index 0c64f54..a01755d 100644 --- a/mocks/valid-indieweb-source.html +++ b/mocks/valid-indieweb-source.html @@ -276,7 +276,7 @@

diff --git a/rest/utils.go b/rest/utils.go index 99011f5..e1b03e1 100644 --- a/rest/utils.go +++ b/rest/utils.go @@ -2,8 +2,10 @@ package rest import ( "encoding/json" + "fmt" "net/http" "net/url" + "strings" ) // mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076 @@ -19,6 +21,18 @@ func Unauthorized(w http.ResponseWriter) { http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) } +// Domain parses the target url to extract the domain as part of the allowed webmention targets. +// This is the same as conf.FetchDomain(wm.Target), only without config, and without error handling. +// Assumes http(s) protocol, which should have been validated by now. +func Domain(target string) string { + withPossibleSubdomain := strings.Split(target, "/")[2] + split := strings.Split(withPossibleSubdomain, ".") + if len(split) == 2 { + return withPossibleSubdomain // that was the extention, not the subdomain. + } + return fmt.Sprintf("%s.%s", split[1], split[2]) +} + func Json(w http.ResponseWriter, data interface{}) { w.WriteHeader(200) bytes, _ := json.MarshalIndent(data, "", " ") diff --git a/rest/utils_test.go b/rest/utils_test.go new file mode 100644 index 0000000..9e69df1 --- /dev/null +++ b/rest/utils_test.go @@ -0,0 +1,41 @@ +package rest + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDomainParseFromTarget(t *testing.T) { + cases := []struct { + label string + target string + expected string + }{ + { + "parse from default http domain", + "http://patat.be/frietjes/zijn/lekker", + "patat.be", + }, + { + "parse from default https domain", + "https://frit.be/patatjes/zijn/lekker", + "frit.be", + }, + { + "parse from default https domain with www subdomain", + "https://www.frit.be/patatjes/zijn/lekker", + "frit.be", + }, + { + "parse from default https domain with some random subdomain", + "https://mayonaise.frit.be/patatjes/zijn/lekker", + "frit.be", + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + assert.Equal(t, tc.expected, Domain(tc.target)) + }) + } +}