implement repost-of microformat + test against brid.gy twitter example

wmio import: also trim name and add extra tests
wmio import: in case of silo domains, use url instead of src
2022-07-01 11:55:34 +02:00 · 2022-06-22 21:28:55 +02:00 · 2022-06-22 21:19:44 +02:00 · 2022-06-22 15:19:53 +02:00 · 2022-06-22 14:09:25 +02:00
13 changed files with 1283 additions and 28 deletions
--- a/app/external/importer.go
+++ b/app/external/importer.go
@ -0,0 +1,73 @@
+package external
+
+import (
+	"brainbaking.com/go-jamming/app/mf"
+	"brainbaking.com/go-jamming/app/notifier"
+	"brainbaking.com/go-jamming/app/webmention/recv"
+	"brainbaking.com/go-jamming/common"
+	"brainbaking.com/go-jamming/db"
+	"brainbaking.com/go-jamming/rest"
+	"github.com/rs/zerolog/log"
+	"os"
+	"reflect"
+)
+
+type Importer interface {
+	TryImport(data []byte) ([]*mf.IndiewebData, error)
+}
+
+type ImportBootstrapper struct {
+	RestClient rest.Client
+	Conf       *common.Config
+	Repo       db.MentionRepo
+}
+
+func (ib *ImportBootstrapper) Import(file string) {
+	bytes, err := os.ReadFile(file)
+	if err != nil {
+		log.Err(err).Msg("Unable to read file")
+		return
+	}
+
+	importers := []Importer{
+		&WebmentionIOImporter{},
+	}
+
+	var convertedData []*mf.IndiewebData
+
+	for _, i := range importers {
+		convertedData, err = i.TryImport(bytes)
+		if err != nil || len(convertedData) == 0 {
+			log.Warn().Str("importType", reflect.TypeOf(i).String()).Msg("Importer failed or returned zero entries")
+		} else {
+			log.Info().Str("importType", reflect.TypeOf(i).String()).Msg("Suitable converter found!")
+			break
+		}
+	}
+
+	if convertedData == nil {
+		log.Fatal().Msg("No suitable importer found for data, aborting import!")
+		return
+	}
+
+	log.Info().Msg("Conversion succeeded, persisting to data storage...")
+	recv := &recv.Receiver{
+		RestClient: ib.RestClient,
+		Conf:       ib.Conf,
+		Repo:       ib.Repo,
+		Notifier:   &notifier.StringNotifier{},
+	}
+
+	for _, wm := range convertedData {
+		mention := mf.Mention{
+			Source: wm.Source,
+			Target: wm.Target,
+		}
+		ib.Conf.AddToWhitelist(mention.SourceDomain())
+
+		recv.ProcessAuthorPicture(wm)
+		recv.ProcessWhitelistedMention(mention, wm)
+	}
+
+	log.Info().Msg("All done, enjoy your go-jammed mentions!")
+}
--- a/app/external/importer_test.go
+++ b/app/external/importer_test.go
@ -0,0 +1,56 @@
+package external
+
+import (
+	"brainbaking.com/go-jamming/common"
+	"brainbaking.com/go-jamming/db"
+	"brainbaking.com/go-jamming/mocks"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"sort"
+	"testing"
+)
+
+var (
+	cnf = &common.Config{
+		BaseURL:                  "http://localhost:1337/",
+		Port:                     1337,
+		Token:                    "miauwkes",
+		AllowedWebmentionSources: []string{"chrisburnell.com"},
+		Blacklist:                []string{},
+		Whitelist:                []string{"chrisburnell.com"},
+	}
+)
+
+func TestImport(t *testing.T) {
+	repo := db.NewMentionRepo(cnf)
+	bootstrapper := ImportBootstrapper{
+		Conf: cnf,
+		Repo: repo,
+		RestClient: &mocks.RestClientMock{
+			// this will make sure each picture GET fails
+			// otherwise this test is REALLY slow. It will fallback to anonymous pictures
+			GetBodyFunc: mocks.RelPathGetBodyFunc("../../../mocks/"),
+		},
+	}
+
+	t.Cleanup(func() {
+		os.Remove("config.json")
+		db.Purge()
+	})
+
+	bootstrapper.Import("../../mocks/external/wmio.json")
+
+	entries := repo.GetAll("chrisburnell.com")
+	assert.Equal(t, 25, len(entries.Data))
+	sort.Slice(entries.Data, func(i, j int) bool {
+		return entries.Data[i].PublishedDate().After(entries.Data[j].PublishedDate())
+	})
+
+	assert.Equal(t, "https://chrisburnell.com/note/1655219889/", entries.Data[0].Source)
+	assert.Equal(t, "/pictures/anonymous", entries.Data[0].Author.Picture)
+	assert.Equal(t, "", entries.Data[10].Name)
+	assert.Equal(t, "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/", entries.Data[20].Content)
+
+	assert.Contains(t, cnf.Whitelist, "jacky.wtf")
+	assert.Contains(t, cnf.Whitelist, "martymcgui.re")
+}
--- a/app/external/webmentionio.go
+++ b/app/external/webmentionio.go
@ -0,0 +1,162 @@
+package external
+
+import (
+	"brainbaking.com/go-jamming/app/mf"
+	"brainbaking.com/go-jamming/common"
+	"brainbaking.com/go-jamming/rest"
+	"encoding/json"
+)
+
+/*
+An example webmention.io JSON mention object:
+{
+	"source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/252048752",
+	"verified": true,
+	"verified_date": "2022-06-10T08:20:16+00:00",
+	"id": 1412862,
+	"private": false,
+	"data": {
+		"author": {
+			"name": "Felipe Sere",
+			"url": "https://twitter.com/felipesere",
+			"photo": "https://webmention.io/avatar/pbs.twimg.com/ca88219f9ffb14d73bfb2bf88450cadb355bdaf56773947f39c752af3883fecf.jpg"
+		},
+		"url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-252048752",
+		"name": null,
+		"content": null,
+		"published": null,
+		"published_ts": null
+	},
+	"activity": {
+		"type": "like",
+		"sentence": "Felipe Sere favorited a tweet https://chrisburnell.com/bowhead/",
+		"sentence_html": "<a href=\"https://twitter.com/felipesere\">Felipe Sere</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+	},
+	"target": "https://chrisburnell.com/bowhead/"
+}
+*/
+type WebmentionIOFile struct {
+	Links []WebmentionIOMention `json:"links"`
+}
+
+type WebmentionIOMention struct {
+	Source       string               `json:"source"`
+	Target       string               `json:"target"`
+	Verified     bool                 `json:"verified"`
+	VerifiedDate string               `json:"verified_date"`
+	ID           int                  `json:"id"`
+	Private      bool                 `json:"private"`
+	Data         WebmentionIOData     `json:"data"`
+	Activity     WebmentionIOActivity `json:"activity"`
+}
+
+type WebmentionIOActivity struct {
+	Type         string `json:"type"`
+	Sentence     string `json:"sentence"`
+	SentenceHtml string `json:"sentence_html"`
+}
+
+type WebmentionIOData struct {
+	Author      WebmentionIOAuthor `json:"author"`
+	Url         string             `json:"url"`
+	Name        string             `json:"name"`
+	Content     string             `json:"content"`
+	Published   string             `json:"published"`
+	PublishedTs int                `json:"published_ts"`
+}
+
+type WebmentionIOAuthor struct {
+	Name  string `json:"name"`
+	Url   string `json:"url"`
+	Photo string `json:"photo"`
+}
+
+type WebmentionIOImporter struct {
+}
+
+func (wmio *WebmentionIOImporter) TryImport(data []byte) ([]*mf.IndiewebData, error) {
+	var mentions WebmentionIOFile
+	err := json.Unmarshal(data, &mentions)
+	if err != nil {
+		return nil, err
+	}
+
+	var converted []*mf.IndiewebData
+	for _, wmiomention := range mentions.Links {
+		converted = append(converted, convert(wmiomention))
+	}
+
+	return converted, nil
+}
+
+func convert(wmio WebmentionIOMention) *mf.IndiewebData {
+	iType := typeOf(wmio)
+	return &mf.IndiewebData{
+		Author: mf.IndiewebAuthor{
+			Name:    wmio.Data.Author.Name,
+			Picture: wmio.Data.Author.Photo,
+		},
+		Name:         nameOf(wmio, iType),
+		Content:      contentOf(wmio, iType),
+		Published:    publishedDate(wmio),
+		Url:          wmio.Data.Url,
+		Source:       sourceOf(wmio),
+		Target:       wmio.Target,
+		IndiewebType: iType,
+	}
+}
+
+// sourceOf returns wmio.Source unless it detects a silo link such as bridgy.
+// In that case, it returns the data URL. This isn't entirely correct, as it technically never was the sender.
+func sourceOf(wmio WebmentionIOMention) string {
+	srcDomain := rest.Domain(wmio.Source)
+	if common.Includes(rest.SiloDomains, srcDomain) {
+		return wmio.Data.Url
+	}
+
+	return wmio.Source
+}
+
+func nameOf(wmio WebmentionIOMention, iType mf.MfType) string {
+	name := wmio.Data.Name
+	if (iType == mf.TypeReply || iType == mf.TypeLike) && wmio.Data.Name == "" {
+		name = wmio.Data.Content
+	}
+	return common.Shorten(name)
+}
+
+func contentOf(wmio WebmentionIOMention, iType mf.MfType) string {
+	content := wmio.Data.Content
+	if iType == mf.TypeReply || (iType == mf.TypeLike && content == "") {
+		content = wmio.Activity.Sentence
+	}
+	return common.Shorten(content)
+}
+
+// typeOf returns the mf.MfType from a wmio mention.
+func typeOf(wmio WebmentionIOMention) mf.MfType {
+	if wmio.Activity.Type == "like" {
+		return mf.TypeLike
+	}
+	if wmio.Activity.Type == "bookmark" {
+		return mf.TypeBookmark
+	}
+	if wmio.Activity.Type == "reply" {
+		return mf.TypeReply
+	}
+	if wmio.Activity.Type == "link" {
+		return mf.TypeLink
+	}
+	return mf.TypeMention
+}
+
+// publishedDate pries out Published or VerifiedDate from wmio with expecrted format: 2022-05-16T09:24:40+01:00
+// This is the same as target format, but validate nonetheless
+func publishedDate(wmio WebmentionIOMention) string {
+	published := wmio.Data.Published
+	if published == "" {
+		published = wmio.VerifiedDate
+	}
+
+	return common.ToTime(published, mf.DateFormatWithTimeZone).Format(mf.DateFormatWithTimeZone)
+}
--- a/app/external/webmentionio_test.go
+++ b/app/external/webmentionio_test.go
@ -0,0 +1,226 @@
+package external
+
+import (
+	"brainbaking.com/go-jamming/app/mf"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestTryImportName(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	cases := []struct {
+		label        string
+		mention      string
+		expectedDate string
+	}{
+		{
+			"Just use name",
+			`{ "links": [ { "data": { "name": "jefke"  }  } ] }`,
+			"jefke",
+		},
+		{
+			"Trim name if longer than 250 chars",
+			`{ "links": [ { "data": { "name": "a;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfj"  }  } ] }`,
+			"a;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;askdfj;alskdfja;dslfkja;dlfkja;ldkfja;ldkfjla;dkfja;ldkjfa;ldkjfl;...",
+		},
+		{
+			"Use content if name is empty in case of reply",
+			`{ "links": [ { "activity": { "type": "reply" }, "data": { "name": "", "content": "wouter"  }  } ] }`,
+			"wouter",
+		},
+		{
+			"Use content if name is empty in case of like",
+			`{ "links": [ { "activity": { "type": "like" }, "data": { "name": "", "content": "wouter"  }  } ] }`,
+			"wouter",
+		},
+		{
+			"Use name if name is not empty in case of reply",
+			`{ "links": [ { "activity": { "type": "reply" }, "data": { "name": "jef", "content": "wouter"  }  } ] }`,
+			"jef",
+		}, {
+			"Use name if name is not empty in case of like",
+			`{ "links": [ { "activity": { "type": "like" }, "data": { "name": "jef", "content": "wouter"  }  } ] }`,
+			"jef",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.label, func(t *testing.T) {
+			res, err := wmio.TryImport([]byte(tc.mention))
+			assert.NoError(t, err)
+
+			assert.Equal(t, tc.expectedDate, res[0].Name)
+		})
+	}
+}
+
+func TestTryImportBridgyUrl(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	cases := []struct {
+		label          string
+		mention        string
+		expectedSource string
+	}{
+		{
+			"conventional source URL does nothing special",
+			`{ "links": [ { "source": "https://brainbaking.com/lolz"  } ] }`,
+			"https://brainbaking.com/lolz",
+		},
+		{
+			"Source URL from brid.gy takes data URL as source instead",
+			`{ "links": [ { "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/252048752", "data": { "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-252048752" } } ] }`,
+			"https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-252048752",
+		},
+		{
+			"Source URL from brid-gy.appspot.com takes URL as data source instead",
+			`{ "links": [ { "source": "https://brid-gy.appspot.com/post/twitter/iamchrisburnell/1103728693648809984", "data": { "url": "https://twitter.com/adactioLinks/status/1103728693648809984" } } ] }`,
+			"https://twitter.com/adactioLinks/status/1103728693648809984",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.label, func(t *testing.T) {
+			res, err := wmio.TryImport([]byte(tc.mention))
+			assert.NoError(t, err)
+
+			assert.Equal(t, tc.expectedSource, res[0].Source)
+		})
+	}
+}
+
+func TestTryImportPublishedDates(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	cases := []struct {
+		label        string
+		mention      string
+		expectedDate string
+	}{
+		{
+			"no dates reverts to first",
+			`{ "links": [ {   } ] }`,
+			time.Time{}.Format(mf.DateFormatWithTimeZone),
+		},
+		{
+			"no published date reverts to verified date",
+			`{ "links": [ { "verified_date": "2022-05-25T14:28:10+00:00"  } ] }`,
+			"2022-05-25T14:28:10+00:00",
+		},
+		{
+			"published date present takes preference over rest",
+			`{ "links": [ { "data": { "published": "2020-01-25T14:28:10+00:00" }, "verified_date": "2022-05-25T14:28:10+00:00"  } ] }`,
+			"2020-01-25T14:28:10+00:00",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.label, func(t *testing.T) {
+			res, err := wmio.TryImport([]byte(tc.mention))
+			assert.NoError(t, err)
+
+			assert.Equal(t, tc.expectedDate, res[0].Published)
+		})
+	}
+}
+
+func TestTryImportErrorIfInvalidFormat(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	mention := `haha`
+
+	_, err := wmio.TryImport([]byte(mention))
+	assert.Error(t, err)
+}
+
+func TestTryImportForLikeWithMissingAuthor(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	mention := `{ "links": [
+{
+            "source": "https://jacky.wtf/2022/5/BRQo",
+            "verified": true,
+            "verified_date": "2022-05-25T14:28:10+00:00",
+            "id": 1404286,
+            "private": false,
+            "data": {
+                "url": "https://jacky.wtf/2022/5/BRQo",
+                "name": null,
+                "content": null,
+                "published": "2022-05-25T14:26:12+00:00",
+                "published_ts": 1653488772
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/",
+                "sentence_html": "<a href=\"https://jacky.wtf/2022/5/BRQo\">someone</a> liked a post <a href=\"https://chrisburnell.com/article/changing-with-the-times/\">https://chrisburnell.com/article/changing-with-the-times/</a>"
+            },
+            "target": "https://chrisburnell.com/article/changing-with-the-times/"
+        }
+] }`
+
+	res, err := wmio.TryImport([]byte(mention))
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	result := res[0]
+
+	assert.Equal(t, "https://chrisburnell.com/article/changing-with-the-times/", result.Target)
+	assert.Equal(t, "https://jacky.wtf/2022/5/BRQo", result.Source)
+
+	assert.Equal(t, mf.TypeLike, result.IndiewebType)
+	assert.Equal(t, "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/", result.Content)
+	assert.Equal(t, "", result.Name)
+	assert.Equal(t, "https://jacky.wtf/2022/5/BRQo", result.Url)
+	assert.Equal(t, "2022-05-25T14:26:12+00:00", result.Published)
+
+	assert.Equal(t, "", result.Author.Name)
+	assert.Equal(t, "", result.Author.Picture)
+}
+
+func TestTryImportForReply(t *testing.T) {
+	wmio := &WebmentionIOImporter{}
+	mention := `{ "links": [
+	{
+		"source": "https://chrisburnell.com/note/1652693080/",
+		"verified": true,
+		"verified_date": "2022-05-16T19:36:52+00:00",
+		"id": 1399408,
+		"private": false,
+		"data": {
+		"author": {
+			"name": "Chris Burnell",
+				"url": "https://chrisburnell.com/",
+				"photo": "https://webmention.io/avatar/chrisburnell.com/ace41559b8d4e8d8189b285d88b1ea2dc6c53056fc512be7d199c0c8cadc53fe.jpg"
+		},
+		"url": "https://chrisburnell.com/note/1652693080/",
+			"name": null,
+			"content": "<p>first!!1!</p>",
+			"published": "2022-05-16T09:24:40+01:00",
+			"published_ts": 1652689480
+	},
+		"activity": {
+		"type": "reply",
+			"sentence": "Chris Burnell commented 'first!!1!' on a post https://chrisburnell.com/guestbook/",
+			"sentence_html": "<a href=\"https://chrisburnell.com/\">Chris Burnell</a> commented 'first!!1!' on a post <a href=\"https://chrisburnell.com/guestbook/\">https://chrisburnell.com/guestbook/</a>"
+	},
+		"rels": {
+		"canonical": "https://chrisburnell.com/note/1652693080/"
+	},
+		"target": "https://chrisburnell.com/guestbook/"
+	}
+] }`
+
+	res, err := wmio.TryImport([]byte(mention))
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	result := res[0]
+
+	assert.Equal(t, "https://chrisburnell.com/guestbook/", result.Target)
+	assert.Equal(t, "https://chrisburnell.com/note/1652693080/", result.Source)
+
+	assert.Equal(t, mf.TypeReply, result.IndiewebType)
+	assert.Equal(t, "Chris Burnell commented 'first!!1!' on a post https://chrisburnell.com/guestbook/", result.Content)
+	assert.Equal(t, "<p>first!!1!</p>", result.Name)
+	assert.Equal(t, "https://chrisburnell.com/note/1652693080/", result.Url)
+	assert.Equal(t, "2022-05-16T09:24:40+01:00", result.Published)
+
+	assert.Equal(t, "Chris Burnell", result.Author.Name)
+	assert.Equal(t, "https://webmention.io/avatar/chrisburnell.com/ace41559b8d4e8d8189b285d88b1ea2dc6c53056fc512be7d199c0c8cadc53fe.jpg", result.Author.Picture)
+}
--- a/app/mf/microformats.go
+++ b/app/mf/microformats.go
@ -100,13 +100,6 @@ func PublishedNow() string {
 	return common.Now().UTC().Format(DateFormatWithTimeZone)
 }

-func shorten(txt string) string {
-	if len(txt) <= 250 {
-		return txt
-	}
-	return txt[:250] + "..."
-}
-
 // Go stuff: entry.Properties["name"][0].(string),
 // JS stuff: hEntry.properties?.name?.[0]
 // The problem: convoluted syntax and no optional chaining!
@ -238,20 +231,27 @@ func DetermineAuthorName(hEntry *microformats.Microformat) string {
 type MfType string

 const (
+	TypeLink     MfType = "link"
+	TypeReply    MfType = "reply"
+	TypeRepost   MfType = "repost"
 	TypeLike     MfType = "like"
 	TypeBookmark MfType = "bookmark"
 	TypeMention  MfType = "mention"
 )

 func Type(hEntry *microformats.Microformat) MfType {
-	likeOf := Str(hEntry, "like-of")
-	if likeOf != "" {
+	hType := Str(hEntry, "like-of")
+	if hType != "" {
 		return TypeLike
 	}
-	bookmarkOf := Str(hEntry, "bookmark-of")
-	if bookmarkOf != "" {
+	hType = Str(hEntry, "bookmark-of")
+	if hType != "" {
 		return TypeBookmark
 	}
+	hType = Str(hEntry, "repost-of")
+	if hType != "" {
+		return TypeRepost
+	}
 	return TypeMention
 }

@ -271,12 +271,12 @@ func Url(hEntry *microformats.Microformat, source string) string {
 func Content(hEntry *microformats.Microformat) string {
 	bridgyTwitterContent := Str(hEntry, "bridgy-twitter-content")
 	if bridgyTwitterContent != "" {
-		return shorten(bridgyTwitterContent)
+		return common.Shorten(bridgyTwitterContent)
 	}
 	summary := Str(hEntry, "summary")
 	if summary != "" {
-		return shorten(summary)
+		return common.Shorten(summary)
 	}
 	contentEntry := Map(hEntry, "content")["value"]
-	return shorten(contentEntry)
+	return common.Shorten(contentEntry)
 }
--- a/app/notifier/stringnotifier.go
+++ b/app/notifier/stringnotifier.go
@ -1,4 +1,4 @@
-package mocks
+package notifier

 import (
 	"brainbaking.com/go-jamming/app/mf"
--- a/app/webmention/recv/receive.go
+++ b/app/webmention/recv/receive.go
@ -28,7 +28,7 @@ var (
 	errPicUnableToDownload          = errors.New("Unable to download author picture")
 	errPicNoRealImage               = errors.New("Downloaded author picture is not a real image")
 	errPicUnableToSave              = errors.New("Unable to save downloaded author picture")
-	errWontDownloadBecauseOfPrivacy = errors.New("Will not save locally because it's form a silo domain")
+	errWontDownloadBecauseOfPrivacy = errors.New("Will not save locally because it's from a silo domain")
 )

 func (recv *Receiver) Receive(wm mf.Mention) {
@ -62,16 +62,16 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {

 	data := microformats.Parse(strings.NewReader(body), wm.SourceUrl())
 	indieweb := recv.convertBodyToIndiewebData(body, wm, data)
-	recv.processAuthorPicture(indieweb)
+	recv.ProcessAuthorPicture(indieweb)

 	if recv.Conf.IsWhitelisted(wm.Source) {
-		recv.processWhitelistedMention(wm, indieweb)
+		recv.ProcessWhitelistedMention(wm, indieweb)
 	} else {
-		recv.processMentionInModeration(wm, indieweb)
+		recv.ProcessMentionInModeration(wm, indieweb)
 	}
 }

-func (recv *Receiver) processMentionInModeration(wm mf.Mention, indieweb *mf.IndiewebData) {
+func (recv *Receiver) ProcessMentionInModeration(wm mf.Mention, indieweb *mf.IndiewebData) {
 	key, err := recv.Repo.InModeration(wm, indieweb)
 	if err != nil {
 		log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to in moderation db")
@ -83,7 +83,7 @@ func (recv *Receiver) processMentionInModeration(wm mf.Mention, indieweb *mf.Ind
 	log.Info().Str("key", key).Msg("OK: Webmention processed, in moderation.")
 }

-func (recv *Receiver) processWhitelistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
+func (recv *Receiver) ProcessWhitelistedMention(wm mf.Mention, indieweb *mf.IndiewebData) {
 	key, err := recv.Repo.Save(wm, indieweb)
 	if err != nil {
 		log.Error().Err(err).Stringer("wm", wm).Msg("Failed to save new mention to db")
@ -95,7 +95,7 @@ func (recv *Receiver) processWhitelistedMention(wm mf.Mention, indieweb *mf.Indi
 	log.Info().Str("key", key).Msg("OK: Webmention processed, in whitelist.")
 }

-func (recv *Receiver) processAuthorPicture(indieweb *mf.IndiewebData) {
+func (recv *Receiver) ProcessAuthorPicture(indieweb *mf.IndiewebData) {
 	if indieweb.Author.Picture != "" {
 		err := recv.saveAuthorPictureLocally(indieweb)
 		if err != nil {
@ -153,7 +153,7 @@ func (recv *Receiver) parseBodyAsNonIndiewebSite(body string, wm mf.Mention) *mf
 // saveAuthorPictureLocally tries to download the author picture and checks if it's valid based on img header.
 // If it succeeds, it alters the picture path to a local /pictures/x one.
 // If it fails, it returns an error.
-// This refuses to download from silo sources such as brid.gy because of privacy concerns.
+// If strict is true, this refuses to download from silo sources such as brid.gy because of privacy concerns.
 func (recv *Receiver) saveAuthorPictureLocally(indieweb *mf.IndiewebData) error {
 	srcDomain := rest.Domain(indieweb.Source)
 	if common.Includes(rest.SiloDomains, srcDomain) {
--- a/app/webmention/recv/receive_test.go
+++ b/app/webmention/recv/receive_test.go
@ -2,6 +2,7 @@ package recv

 import (
 	"brainbaking.com/go-jamming/app/mf"
+	"brainbaking.com/go-jamming/app/notifier"
 	"brainbaking.com/go-jamming/db"
 	"encoding/json"
 	"errors"
@ -144,7 +145,7 @@ func TestReceive(t *testing.T) {
 			json: `{"author":{"name":"Jamie Tanna","picture":"/pictures/brainbaking.com"},"name":"","content":"Recommended read:\nThe IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions\nhttps://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/","published":"2021-03-15T12:42:00+00:00","url":"https://brainbaking.com/mf2/2021/03/1bkre/","type":"bookmark","source":"https://brainbaking.com/valid-bridgy-twitter-source.html","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag"}`,
 		},
 		{
-			label: "receive a brid.gy Webmention like",
+			label: "receive a brid.gy (Mastodon) Webmention like",
 			wm: mf.Mention{
 				Source: "https://brainbaking.com/valid-bridgy-like.html",
 				// wrapped in a a class="u-like-of" tag
@ -154,7 +155,17 @@ func TestReceive(t *testing.T) {
 			json: `{"author":{"name":"Stampeding Longhorn","picture":"/pictures/brainbaking.com"},"name":"","content":"","published":"2020-01-01T12:30:00+00:00","url":"https://chat.brainbaking.com/notice/A4nx1rFwKUJYSe4TqK#favorited-by-A4nwg4LYyh4WgrJOXg","type":"like","source":"https://brainbaking.com/valid-bridgy-like.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`,
 		},
 		{
-			label: "receive a brid.gy Webmention that has a url and photo without value",
+			label: "receive a brid.gy (Twitter) Webmention repost",
+			wm: mf.Mention{
+				Source: "https://brainbaking.com/valid-bridgy-twitter-repost.html",
+				// wrapped in a a class="u-like-of" tag
+				Target: "https://brainbaking.com/valid-indieweb-target.html",
+			},
+			// no dates in bridgy-to-mastodon likes...
+			json: `{"author":{"name":"cartocalypse.tif","picture":"/pictures/brainbaking.com"},"name":"My quest in creating a Google Maps clone\n\n            chringel.dev/2022/06/creati…","content":"My quest in creating a Google Maps clone\n\n            chringel.dev/2022/06/creati…","published":"2022-06-21T06:23:53+00:00","url":"https://twitter.com/cartocalypse/status/1539131976879308800","type":"repost","source":"https://brainbaking.com/valid-bridgy-twitter-repost.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`,
+		},
+		{
+			label: "receive a brid.gy (Mastodon) Webmention that has a url and photo without value",
 			wm: mf.Mention{
 				Source: "https://brainbaking.com/valid-bridgy-source.html",
 				Target: "https://brainbaking.com/valid-indieweb-target.html",
@ -200,6 +211,7 @@ func TestReceive(t *testing.T) {
 				RestClient: &mocks.RestClientMock{
 					GetBodyFunc: mocks.RelPathGetBodyFunc("../../../mocks/"),
 				},
+				Notifier: &notifier.StringNotifier{},
 			}

 			receiver.Receive(tc.wm)
@ -255,7 +267,7 @@ func TestReceiveFromNotInWhitelistSavesInModerationAndNotifies(t *testing.T) {
 	}
 	repo := db.NewMentionRepo(cnf)
 	t.Cleanup(db.Purge)
-	notifierMock := &mocks.StringNotifier{
+	notifierMock := &notifier.StringNotifier{
 		Conf:   cnf,
 		Output: "",
 	}
@ -415,7 +427,7 @@ func TestProcessAuthorPictureAnonymizesIfEmpty(t *testing.T) {
 			Picture: "",
 		},
 	}
-	recv.processAuthorPicture(indieweb)
+	recv.ProcessAuthorPicture(indieweb)

 	assert.Equal(t, "/pictures/anonymous", indieweb.Author.Picture)
 }
--- a/common/strings.go
+++ b/common/strings.go
@ -0,0 +1,8 @@
+package common
+
+func Shorten(txt string) string {
+	if len(txt) <= 250 {
+		return txt
+	}
+	return txt[:250] + "..."
+}
--- a/main.go
+++ b/main.go
@ -1,8 +1,10 @@
 package main

 import (
+	"brainbaking.com/go-jamming/app/external"
 	"brainbaking.com/go-jamming/common"
 	"brainbaking.com/go-jamming/db"
+	"brainbaking.com/go-jamming/rest"
 	"flag"
 	"os"

@ -18,12 +20,14 @@ func main() {
 	verboseFlag := flag.Bool("verbose", false, "Verbose mode (pretty print log, debug level)")
 	migrateFlag := flag.Bool("migrate", false, "Run migration scripts for the DB and exit.")
 	blacklist := flag.String("blacklist", "", "Blacklist a domain name (also cleans spam from DB)")
+	importFile := flag.String("import", "", "Import mentions from an external source (i.e. webmention.io)")
 	flag.Parse()
 	blacklisting := len(*blacklist) > 1
+	importing := len(*importFile) > 1

 	// logs by default to Stderr (/var/log/syslog). Rolling files possible via lumberjack.
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	if *verboseFlag || *migrateFlag || blacklisting {
+	if *verboseFlag || *migrateFlag || blacklisting || importing {
 		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
@ -38,10 +42,27 @@ func main() {
 		os.Exit(0)
 	}

+	if importing {
+		importWebmentionFile(*importFile)
+		os.Exit(0)
+	}
+
 	log.Debug().Msg("Let's a go!")
 	app.Start()
 }

+func importWebmentionFile(file string) {
+	log.Info().Str("file", file).Msg("Starting import...")
+
+	config := common.Configure()
+	bootstrapper := external.ImportBootstrapper{
+		RestClient: &rest.HttpClient{},
+		Conf:       config,
+		Repo:       db.NewMentionRepo(config),
+	}
+	bootstrapper.Import(file)
+}
+
 func blacklistDomain(domain string) {
 	log.Info().Str("domain", domain).Msg("Blacklisting...")
 	config := common.Configure()
--- a/mocks/external/wmio.json
+++ b/mocks/external/wmio.json
@ -0,0 +1,636 @@
+{
+    "links": [
+        {
+            "source": "https://chrisburnell.com/note/1655219889/",
+            "verified": true,
+            "verified_date": "2022-06-14T18:28:55+00:00",
+            "id": 1416849,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Chris Burnell",
+                    "url": "https://chrisburnell.com/",
+                    "photo": "https://webmention.io/avatar/chrisburnell.com/ace41559b8d4e8d8189b285d88b1ea2dc6c53056fc512be7d199c0c8cadc53fe.jpg"
+                },
+                "url": "https://chrisburnell.com/note/1655219889/",
+                "name": null,
+                "content": "<p>Given a couple of assumptions about how the data is returned from the server, this change allows you to use different Webmention servers, hopefully lessening the <q>inherent</q></p>\n<p>This change does mean that there are now two more <strong>required</strong> fields that need to be passed to the plugin when invoking it in your <code>.eleventy.js</code> file with <code>addPlugin()</code>:</p>\n<pre><code><span>const</span> pluginWebmentions <span>=</span> <span>require</span><span>(</span><span>\"@chrisburnell/eleventy-cache-webmentions\"</span><span>)</span><br /><br />module<span>.</span><span>exports</span> <span>=</span> <span>function</span><span>(</span><span>eleventyConfig</span><span>)</span> <span>{</span><br />    eleventyConfig<span>.</span><span>addPlugin</span><span>(</span>pluginWebmentions<span>,</span> <span>{</span><br /><span>// these 3 fields are all required!</span><br /><span>domain</span><span>:</span> <span>\"https://example.com\"</span><span>,</span><br /><span>feed</span><span>:</span> <span>\"https://webmention.io/api/mentions.jf2?domain=example.com&amp;token=${process.env.WEBMENTION_IO_TOKEN}&amp;per-page=9001\"</span><span>,</span><br /><span>key</span><span>:</span> <span>\"children\"</span><br /><span>}</span><span>)</span><br /><span>}</span></code></pre>\n<p>The above example, which I’m hoping is simple to update in existing configs, outlines how to use the plugin with <a href=\"https://webmention.io\">Webmention.io</a>.</p>",
+                "published": "2022-06-14T16:18:09+01:00",
+                "published_ts": 1655219889
+            },
+            "activity": {
+                "type": "reply",
+                "sentence": "Chris Burnell commented 'Given a couple of assumptions about how the data is returned from the server, th...' on a post https://chrisburnell.com/eleventy-cache-webmentions/",
+                "sentence_html": "<a href=\"https://chrisburnell.com/\">Chris Burnell</a> commented 'Given a couple of assumptions about how the data is returned from the server, th...' on a post <a href=\"https://chrisburnell.com/eleventy-cache-webmentions/\">https://chrisburnell.com/eleventy-cache-webmentions/</a>"
+            },
+            "rels": {
+                "canonical": "https://chrisburnell.com/note/1655219889/"
+            },
+            "target": "https://chrisburnell.com/eleventy-cache-webmentions/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/252048752",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:16+00:00",
+            "id": 1412862,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Felipe Sere",
+                    "url": "https://twitter.com/felipesere",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/ca88219f9ffb14d73bfb2bf88450cadb355bdaf56773947f39c752af3883fecf.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-252048752",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Felipe Sere favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/felipesere\">Felipe Sere</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/36324447",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:15+00:00",
+            "id": 1412860,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Brian Milne",
+                    "url": "https://twitter.com/BrianDMilne",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/838e9248b675037b510b5ba6e0020933fd47cf5350ae6793a291767f77ccef6a.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-36324447",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Brian Milne favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/BrianDMilne\">Brian Milne</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/685233",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:16+00:00",
+            "id": 1412861,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Jeff Byrnes 🚰",
+                    "url": "https://twitter.com/thejeffbyrnes",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/001c536d4ffe238ac976935b137b07e4df50a12abe6e99b1233207a9beb52c23.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-685233",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Jeff Byrnes 🚰 favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/thejeffbyrnes\">Jeff Byrnes 🚰</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/17018537",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:14+00:00",
+            "id": 1412858,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "tanangular",
+                    "url": "https://twitter.com/cooldezign",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/99eaf005a189df113a75c71a394d9709ec60f0fa2964112a886c52a3cbf7953b.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-17018537",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "tanangular favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/cooldezign\">tanangular</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/85053418",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:15+00:00",
+            "id": 1412859,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "JamCow",
+                    "url": "https://twitter.com/jamcow",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/dace09fd6486df0064057e6ee4d8b06a23728f8e50e5c68b2edff3f0a1c1ba9b.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-85053418",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "JamCow favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/jamcow\">JamCow</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/889737980",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:13+00:00",
+            "id": 1412856,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Mike Riethmuller",
+                    "url": "https://twitter.com/MikeRiethmuller",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/e26c69ef63c66748a1153598f78141b5648a4dc186888369f368f00ab780708b.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-889737980",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Mike Riethmuller favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/MikeRiethmuller\">Mike Riethmuller</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/347785341",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:14+00:00",
+            "id": 1412857,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "image enthusiast | #GenocideOfUkrainians 🌻",
+                    "url": "https://twitter.com/sasvstheworld",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/720151d67e3b75b74fbc9f646c0bb8c5fcb64582bbbfd9aba853c1c1bd049445.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-347785341",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "image enthusiast | #GenocideOfUkrainians 🌻 favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/sasvstheworld\">image enthusiast | #GenocideOfUkrainians 🌻</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/8252062",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:13+00:00",
+            "id": 1412855,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Jennifer Ecker",
+                    "url": "https://twitter.com/drjecker",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/38bb76840d26f4e5b4b091363fdc62c0bb5dbf48c3dbba526b33a5a513545eae.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-8252062",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Jennifer Ecker favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/drjecker\">Jennifer Ecker</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/28527216",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:11+00:00",
+            "id": 1412853,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Bill Rafferty",
+                    "url": "https://twitter.com/bulltwitting",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/cf46db3d996381b6a2f977d7106507f9f7375e1964243f4192b16ddf856a3806.png"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-28527216",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Bill Rafferty favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/bulltwitting\">Bill Rafferty</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/432888518",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:12+00:00",
+            "id": 1412854,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Ryan Mulligan",
+                    "url": "https://twitter.com/hexagoncircle",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/83ecdd1f7b2d6a7a90e86802aa1ae6c2e0d2819b83db984892f9c0be96584092.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-432888518",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Ryan Mulligan favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/hexagoncircle\">Ryan Mulligan</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/205775928",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:10+00:00",
+            "id": 1412852,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Michelle Barker",
+                    "url": "https://twitter.com/MicheBarks",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/46365cd70695ca723991011a6c5a8058ec0108b4b9b31244511b9dff925663c2.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-205775928",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Michelle Barker favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/MicheBarks\">Michelle Barker</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/1319278286",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:07+00:00",
+            "id": 1412851,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Stephen Margheim",
+                    "url": "https://twitter.com/fractaledmind",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/39c01731bd41a8acfdf8755cee4a23396783673c37a39499079479d1891b507e.png"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-1319278286",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Stephen Margheim favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/fractaledmind\">Stephen Margheim</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/1007024772209758213",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:06+00:00",
+            "id": 1412849,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Jack Howells",
+                    "url": "https://twitter.com/jhwlls",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/4874f2eea87d1ee675c7a5589f039fbca97561b3dbed1892dfaaac8b25bcea15.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-1007024772209758213",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Jack Howells favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/jhwlls\">Jack Howells</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/17061996",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:06+00:00",
+            "id": 1412850,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "maarten brouwers 🇺🇳 / murb@todon.nl",
+                    "url": "https://twitter.com/murb",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/d9a3ef93825ec75804176ca9b05522d87e66618f994515cf9122565c05fef1b1.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-17061996",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "maarten brouwers 🇺🇳 / murb@todon.nl favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/murb\">maarten brouwers 🇺🇳 / murb@todon.nl</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/2260815740",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:05+00:00",
+            "id": 1412847,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Jake",
+                    "url": "https://twitter.com/JakeDChampion",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/55452fdbb1b010495f28472fab8127f5c5d5ed7aa4f83162020af3774bb163a8.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-2260815740",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Jake favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/JakeDChampion\">Jake</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/98734097",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:05+00:00",
+            "id": 1412848,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Andy Bell",
+                    "url": "https://twitter.com/hankchizljaw",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/ad7c495b44c43890f231501fcbe4be1c6497c47db16ed3f60a26c9ff440c38af.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-98734097",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Andy Bell favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/hankchizljaw\">Andy Bell</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/15608209",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:04+00:00",
+            "id": 1412846,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Christopher Adams",
+                    "url": "https://twitter.com/koitaki",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/b9ad3e0e7eb4163ba9fb3e79e8a41d3766556e89b2dbb88a392884995718cebf.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-15608209",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Christopher Adams favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/koitaki\">Christopher Adams</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/44765646",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:04+00:00",
+            "id": 1412845,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Jason Lee Cooksey",
+                    "url": "https://twitter.com/jasonleecooksey",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/dbe7a6f4c8e71103c6e541c2c2e779ff92ca040b890e45e127d851e70cee0cf0.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-44765646",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Jason Lee Cooksey favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/jasonleecooksey\">Jason Lee Cooksey</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://brid.gy/like/twitter/iamchrisburnell/1298550501307486208/1241894042557972480",
+            "verified": true,
+            "verified_date": "2022-06-10T08:20:02+00:00",
+            "id": 1412844,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Matt DeCamp",
+                    "url": "https://twitter.com/mpdecamp",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/019a87e85b2d4b3f228b07b12cf1b0c1d278b36af31cc8edf9fee9e1dccf1b8f.jpg"
+                },
+                "url": "https://twitter.com/iamchrisburnell/status/1298550501307486208#favorited-by-1241894042557972480",
+                "name": null,
+                "content": null,
+                "published": null,
+                "published_ts": null
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Matt DeCamp favorited a tweet https://chrisburnell.com/bowhead/",
+                "sentence_html": "<a href=\"https://twitter.com/mpdecamp\">Matt DeCamp</a> favorited a tweet <a href=\"https://chrisburnell.com/bowhead/\">https://chrisburnell.com/bowhead/</a>"
+            },
+            "target": "https://chrisburnell.com/bowhead/"
+        },
+        {
+            "source": "https://jacky.wtf/2022/5/BRQo",
+            "verified": true,
+            "verified_date": "2022-05-25T14:28:10+00:00",
+            "id": 1404286,
+            "private": false,
+            "data": {
+                "url": "https://jacky.wtf/2022/5/BRQo",
+                "name": null,
+                "content": null,
+                "published": "2022-05-25T14:26:12+00:00",
+                "published_ts": 1653488772
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "https://jacky.wtf/2022/5/BRQo liked a post https://chrisburnell.com/article/changing-with-the-times/",
+                "sentence_html": "<a href=\"https://jacky.wtf/2022/5/BRQo\">someone</a> liked a post <a href=\"https://chrisburnell.com/article/changing-with-the-times/\">https://chrisburnell.com/article/changing-with-the-times/</a>"
+            },
+            "target": "https://chrisburnell.com/article/changing-with-the-times/"
+        },
+        {
+            "source": "https://chrisburnell.com/note/1652693080/",
+            "verified": true,
+            "verified_date": "2022-05-16T19:36:52+00:00",
+            "id": 1399408,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Chris Burnell",
+                    "url": "https://chrisburnell.com/",
+                    "photo": "https://webmention.io/avatar/chrisburnell.com/ace41559b8d4e8d8189b285d88b1ea2dc6c53056fc512be7d199c0c8cadc53fe.jpg"
+                },
+                "url": "https://chrisburnell.com/note/1652693080/",
+                "name": null,
+                "content": "<p>first!!1!</p>",
+                "published": "2022-05-16T09:24:40+01:00",
+                "published_ts": 1652689480
+            },
+            "activity": {
+                "type": "reply",
+                "sentence": "Chris Burnell commented 'first!!1!' on a post https://chrisburnell.com/guestbook/",
+                "sentence_html": "<a href=\"https://chrisburnell.com/\">Chris Burnell</a> commented 'first!!1!' on a post <a href=\"https://chrisburnell.com/guestbook/\">https://chrisburnell.com/guestbook/</a>"
+            },
+            "rels": {
+                "canonical": "https://chrisburnell.com/note/1652693080/"
+            },
+            "target": "https://chrisburnell.com/guestbook/"
+        },
+        {
+            "source": "https://chrisburnell.com/note/1652634000/",
+            "verified": true,
+            "verified_date": "2022-05-15T18:32:00+00:00",
+            "id": 1398942,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Chris Burnell",
+                    "url": "https://chrisburnell.com/",
+                    "photo": "https://webmention.io/avatar/chrisburnell.com/ace41559b8d4e8d8189b285d88b1ea2dc6c53056fc512be7d199c0c8cadc53fe.jpg"
+                },
+                "url": "https://chrisburnell.com/note/1652634000/",
+                "name": null,
+                "content": "<p>Happy to say the sparklines are going well—pushed an update yesterday, in fact, to allow them to inherit the color of wherever they are in the DOM for the colour of the line itself.</p>\n<p>Still working on posting a bit more! 😅</p>",
+                "published": "2022-05-15T18:00:00+01:00",
+                "published_ts": 1652634000
+            },
+            "activity": {
+                "type": "reply",
+                "sentence": "Chris Burnell commented 'Happy to say the sparklines are going well—pushed an update yesterday, in fact, ...' on a post https://chrisburnell.com/note/1526371395/",
+                "sentence_html": "<a href=\"https://chrisburnell.com/\">Chris Burnell</a> commented 'Happy to say the sparklines are going well—pushed an update yesterday, in fact, ...' on a post <a href=\"https://chrisburnell.com/note/1526371395/\">https://chrisburnell.com/note/1526371395/</a>"
+            },
+            "rels": {
+                "canonical": "https://chrisburnell.com/note/1652634000/"
+            },
+            "target": "https://chrisburnell.com/note/1526371395/"
+        },
+        {
+            "source": "https://martymcgui.re/2022/05/15/004058/",
+            "verified": true,
+            "verified_date": "2022-05-15T04:42:32+00:00",
+            "id": 1398724,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Marty McGuire",
+                    "url": "https://martymcgui.re/",
+                    "photo": "https://webmention.io/avatar/martymcgui.re/f750b9918a92f3dc86d15d8fefad4a06c20a829ae950e18dfc2c8b9a4b26b422.jpg"
+                },
+                "url": "https://martymcgui.re/2022/05/15/004058/",
+                "name": null,
+                "content": null,
+                "published": "2022-05-15T00:40:58-04:00",
+                "published_ts": 1652589658
+            },
+            "activity": {
+                "type": "like",
+                "sentence": "Marty McGuire liked a post https://chrisburnell.com/article/changing-with-the-times/",
+                "sentence_html": "<a href=\"https://martymcgui.re/\">Marty McGuire</a> liked a post <a href=\"https://chrisburnell.com/article/changing-with-the-times/\">https://chrisburnell.com/article/changing-with-the-times/</a>"
+            },
+            "rels": {
+                "canonical": "https://martymcgui.re/2022/05/15/004058/"
+            },
+            "target": "https://chrisburnell.com/article/changing-with-the-times/"
+        },
+        {
+            "source": "https://brid.gy/post/twitter/iamchrisburnell/1524869209804660750",
+            "verified": true,
+            "verified_date": "2022-05-13T00:36:53+00:00",
+            "id": 1397468,
+            "private": false,
+            "data": {
+                "author": {
+                    "name": "Adactio Links",
+                    "url": "https://twitter.com/adactioLinks",
+                    "photo": "https://webmention.io/avatar/pbs.twimg.com/8e0a805eae6791b05c23541931a123a5446393bb9929fcdd274f7e5c33d7792f.jpeg"
+                },
+                "url": "https://twitter.com/adactioLinks/status/1524869209804660750",
+                "name": null,
+                "content": "Changing with the times · Chris Burnell <a href=\"https://chrisburnell.com/article/changing-with-the-times/\">chrisburnell.com/article/changi…</a>",
+                "published": "2022-05-12T21:48:44+00:00",
+                "published_ts": 1652392124
+            },
+            "activity": {
+                "type": "link",
+                "sentence": "Adactio Links posted 'Changing with the times · Chris Burnell chrisburnell.com/article/changi…' linking to https://chrisburnell.com/article/changing-with-the-times/",
+                "sentence_html": "<a href=\"https://twitter.com/adactioLinks\">Adactio Links</a> posted 'Changing with the times · Chris Burnell chrisburnell.com/article/changi…' linking to <a href=\"https://chrisburnell.com/article/changing-with-the-times/\">https://chrisburnell.com/article/changing-with-the-times/</a>"
+            },
+            "target": "https://chrisburnell.com/article/changing-with-the-times/"
+        }
+    ]
+}
--- a/mocks/valid-bridgy-twitter-repost.html
+++ b/mocks/valid-bridgy-twitter-repost.html
@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="refresh" content="0;url=https://twitter.com/cartocalypse/status/1539131976879308800">
+    <title>My quest in creating a Google Maps clone
+
+        chringel.dev/2022/06/creati…</title>
+    <style type="text/css">
+        body {
+            display: none;
+            font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
+        }
+        .p-uid {
+            display: none;
+        }
+        .u-photo {
+            max-width: 50px;
+            border-radius: 4px;
+        }
+        .e-content {
+            margin-top: 10px;
+            font-size: 1.3em;
+        }
+    </style>
+</head>
+<article class="h-entry">
+    <span class="p-uid">tag:twitter.com,2013:1539131976879308800</span>
+
+    <time class="dt-published" datetime="2022-06-21T06:23:53+00:00">2022-06-21T06:23:53+00:00</time>
+
+    <span class="p-author h-card">
+    <data class="p-uid" value="tag:twitter.com,2013:cartocalypse"></data>
+<data class="p-numeric-id" value="1710706561"></data>
+    <a class="p-name u-url" href="https://twitter.com/cartocalypse">cartocalypse.tif</a>
+<a class="u-url" href="https://hannes.enjoys.it/blog/"></a>
+<a class="u-url" href="https://m.youtube.com/watch?v=MEBK3ZUn0c4"></a>
+    <span class="p-nickname">cartocalypse</span>
+    <img class="u-photo" src="https://brainbaking.com/picture.jpg" alt="" />
+  </span>
+
+    <a class="u-url" href="https://twitter.com/cartocalypse/status/1539131976879308800">https://twitter.com/cartocalypse/status/1539131976879308800</a>
+    <div class="e-content p-name">
+
+        <div style="white-space: pre">My quest in creating a Google Maps clone
+
+            chringel.dev/2022/06/creati…</div>
+    </div>
+
+
+
+
+
+
+    <a class="u-repost-of" href="https://twitter.com/DeEgge/status/1538902082698231808"></a>
+    <a class="u-repost-of" href="https://brainbaking.com/valid-indieweb-target.html"></a>
+
+</article>
+
+</html>
--- a/rest/utils.go
+++ b/rest/utils.go
@ -61,6 +61,7 @@ var (
 	// These are privacy issues and will be anonymized as such.
 	SiloDomains = []string{
 		"brid.gy",
+		"brid-gy.appspot.com",
 		"twitter.com",
 		"facebook.com",
 		"indieweb.social",
Author	SHA1	Message	Date
Wouter Groeneveld	4c2326b887	implement repost-of microformat + test against brid.gy twitter example	2022-07-01 11:55:34 +02:00
Wouter Groeneveld	92252f8bc9	wmio import: also trim name and add extra tests	2022-06-22 21:28:55 +02:00
Wouter Groeneveld	6503037098	wmio import: in case of silo domains, use url instead of src	2022-06-22 21:19:44 +02:00
Wouter Groeneveld	989495a18e	webmention.io import functionality implementation	2022-06-22 15:19:53 +02:00
Wouter Groeneveld	fb5ba20292	webmention.io import functionality funadmentals	2022-06-22 14:09:25 +02:00