From ddd465ce920ef5130ed7b22cb1a6d141687de9ac Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Fri, 9 Apr 2021 14:21:25 +0200 Subject: [PATCH] refactored package design to avoid circular deps --- app/mf/mention.go | 28 ++++++++ app/{webmention => mf}/microformats.go | 52 +++++++------- app/pingback/handler.go | 7 +- app/pingback/send/send.go | 9 +++ app/webmention/handler.go | 6 +- app/webmention/{ => receive}/receive.go | 71 +++++++------------- app/webmention/{ => receive}/receive_test.go | 41 +++++------ app/webmention/send/send.go | 10 +++ common/time.go | 6 ++ mocks/restclient.go | 4 +- 10 files changed, 135 insertions(+), 99 deletions(-) create mode 100644 app/mf/mention.go rename app/{webmention => mf}/microformats.go (60%) create mode 100644 app/pingback/send/send.go rename app/webmention/{ => receive}/receive.go (59%) rename app/webmention/{ => receive}/receive_test.go (91%) create mode 100644 app/webmention/send/send.go create mode 100644 common/time.go diff --git a/app/mf/mention.go b/app/mf/mention.go new file mode 100644 index 0000000..87d04d6 --- /dev/null +++ b/app/mf/mention.go @@ -0,0 +1,28 @@ +package mf + +import ( + "crypto/md5" + "fmt" + "github.com/wgroeneveld/go-jamming/common" + "net/url" +) + +type Mention struct { + Source string + Target string +} + +func (wm *Mention) String() string { + return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target) +} + +func (wm *Mention) AsPath(conf *common.Config) string { + filename := fmt.Sprintf("%x", md5.Sum([]byte("source=" + wm.Source+ ",target=" + wm.Target))) + domain, _ := conf.FetchDomain(wm.Target) + return conf.DataPath + "/" + domain + "/" + filename + ".json" +} + +func (wm *Mention) SourceUrl() *url.URL { + url, _ := url.Parse(wm.Source) + return url +} diff --git a/app/webmention/microformats.go b/app/mf/microformats.go similarity index 60% rename from app/webmention/microformats.go rename to app/mf/microformats.go index bc2d7fe..a3636bb 100644 --- a/app/webmention/microformats.go +++ b/app/mf/microformats.go @@ -1,22 +1,23 @@ -package webmention +package mf import ( "strings" "time" "willnorris.com/go/microformats" + "github.com/wgroeneveld/go-jamming/common" ) const ( DateFormat = "2006-01-02T15:04:05" ) -type indiewebAuthor struct { +type IndiewebAuthor struct { Name string `json:"name"` Picture string `json:"picture"` } -type indiewebData struct { - Author indiewebAuthor `json:"author"` +type IndiewebData struct { + Author IndiewebAuthor `json:"author"` Name string `json:"name"` Content string `json:"content"` Published string `json:"published"` @@ -26,9 +27,8 @@ type indiewebData struct { Target string `json:"target"` } -var now = time.Now -func publishedNow(utcOffset int) string { - return now().UTC().Add(time.Duration(utcOffset) * time.Minute).Format("2006-01-02T15:04:05") +func PublishedNow(utcOffset int) string { + return common.Now().UTC().Add(time.Duration(utcOffset) * time.Minute).Format("2006-01-02T15:04:05") } func shorten(txt string) string { @@ -41,7 +41,7 @@ func shorten(txt string) string { // Go stuff: entry.Properties["name"][0].(string), // JS stuff: hEntry.properties?.name?.[0] // The problem: convoluted syntax and no optional chaining! -func mfStr(mf *microformats.Microformat, key string) string { +func Str(mf *microformats.Microformat, key string) string { val := mf.Properties[key] if len(val) == 0 { return "" @@ -60,7 +60,7 @@ func mfStr(mf *microformats.Microformat, key string) string { return str } -func mfMap(mf *microformats.Microformat, key string) map[string]string { +func Map(mf *microformats.Microformat, key string) map[string]string { val := mf.Properties[key] if len(val) == 0 { return map[string]string{} @@ -72,7 +72,7 @@ func mfMap(mf *microformats.Microformat, key string) map[string]string { return mapVal } -func mfProp(mf *microformats.Microformat, key string) *microformats.Microformat { +func Prop(mf *microformats.Microformat, key string) *microformats.Microformat { val := mf.Properties[key] if len(val) == 0 { return µformats.Microformat{ @@ -82,28 +82,28 @@ func mfProp(mf *microformats.Microformat, key string) *microformats.Microformat return val[0].(*microformats.Microformat) } -func determinePublishedDate(hEntry *microformats.Microformat, utcOffset int) string { - publishedDate := mfStr(hEntry, "published") +func DeterminePublishedDate(hEntry *microformats.Microformat, utcOffset int) string { + publishedDate := Str(hEntry, "published") if publishedDate == "" { - return publishedNow(utcOffset) + return PublishedNow(utcOffset) } return publishedDate } -func determineAuthorName(hEntry *microformats.Microformat) string { - authorName := mfStr(mfProp(hEntry, "author"), "name") +func DetermineAuthorName(hEntry *microformats.Microformat) string { + authorName := Str(Prop(hEntry, "author"), "name") if authorName == "" { - return mfProp(hEntry, "author").Value + return Prop(hEntry, "author").Value } return authorName } -func determineMfType(hEntry *microformats.Microformat) string { - likeOf := mfStr(hEntry, "like-of") +func DetermineType(hEntry *microformats.Microformat) string { + likeOf := Str(hEntry, "like-of") if likeOf != "" { return "like" } - bookmarkOf := mfStr(hEntry, "bookmark-of") + bookmarkOf := Str(hEntry, "bookmark-of") if bookmarkOf != "" { return "bookmark" } @@ -111,27 +111,27 @@ func determineMfType(hEntry *microformats.Microformat) string { } // Mastodon uids start with "tag:server", but we do want indieweb uids from other sources -func determineUrl(hEntry *microformats.Microformat, source string) string { - uid := mfStr(hEntry, "uid") +func DetermineUrl(hEntry *microformats.Microformat, source string) string { + uid := Str(hEntry, "uid") if uid != "" && strings.HasPrefix(uid, "http") { return uid } - url := mfStr(hEntry, "url") + url := Str(hEntry, "url") if url != "" { return url } return source } -func determineContent(hEntry *microformats.Microformat) string { - bridgyTwitterContent := mfStr(hEntry, "bridgy-twitter-content") +func DetermineContent(hEntry *microformats.Microformat) string { + bridgyTwitterContent := Str(hEntry, "bridgy-twitter-content") if bridgyTwitterContent != "" { return shorten(bridgyTwitterContent) } - summary := mfStr(hEntry, "summary") + summary := Str(hEntry, "summary") if summary != "" { return shorten(summary) } - contentEntry := mfMap(hEntry, "content")["value"] + contentEntry := Map(hEntry, "content")["value"] return shorten(contentEntry) } \ No newline at end of file diff --git a/app/pingback/handler.go b/app/pingback/handler.go index 794a80f..a3cccad 100644 --- a/app/pingback/handler.go +++ b/app/pingback/handler.go @@ -4,7 +4,8 @@ package pingback import ( "encoding/xml" "github.com/rs/zerolog/log" - "github.com/wgroeneveld/go-jamming/app/webmention" + "github.com/wgroeneveld/go-jamming/app/mf" + "github.com/wgroeneveld/go-jamming/app/webmention/receive" "github.com/wgroeneveld/go-jamming/rest" "io/ioutil" "net/http" @@ -30,11 +31,11 @@ func HandlePost(conf *common.Config) http.HandlerFunc { return } - wm := webmention.Mention{ + wm := mf.Mention{ Source: rpc.Source(), Target: rpc.Target(), } - receiver := webmention.Receiver{ + receiver := receive.Receiver{ RestClient: &rest.HttpClient{}, Conf: conf, } diff --git a/app/pingback/send/send.go b/app/pingback/send/send.go new file mode 100644 index 0000000..8fa1c4b --- /dev/null +++ b/app/pingback/send/send.go @@ -0,0 +1,9 @@ +package send + +import ( + "github.com/wgroeneveld/go-jamming/app/mf" +) + +func SendPingbackToEndpoint(endpoint string, mention mf.Mention) { + // do stuff +} diff --git a/app/webmention/handler.go b/app/webmention/handler.go index 532ad70..b1429b5 100644 --- a/app/webmention/handler.go +++ b/app/webmention/handler.go @@ -2,6 +2,8 @@ package webmention import ( + "github.com/wgroeneveld/go-jamming/app/mf" + "github.com/wgroeneveld/go-jamming/app/webmention/receive" "net/http" "fmt" @@ -37,11 +39,11 @@ func HandlePost(conf *common.Config) http.HandlerFunc { return } - wm := Mention{ + wm := mf.Mention{ Source: r.FormValue("source"), Target: target, } - recv := &Receiver{ + recv := &receive.Receiver{ RestClient: httpClient, Conf: conf, } diff --git a/app/webmention/receive.go b/app/webmention/receive/receive.go similarity index 59% rename from app/webmention/receive.go rename to app/webmention/receive/receive.go index 93e2433..5ceea46 100644 --- a/app/webmention/receive.go +++ b/app/webmention/receive/receive.go @@ -1,15 +1,13 @@ -package webmention +package receive import ( - "crypto/md5" "encoding/json" - "fmt" + "github.com/wgroeneveld/go-jamming/app/mf" "github.com/wgroeneveld/go-jamming/common" "github.com/wgroeneveld/go-jamming/rest" "io/fs" "io/ioutil" - "net/url" "os" "regexp" "strings" @@ -18,25 +16,6 @@ import ( "willnorris.com/go/microformats" ) -type Mention struct { - Source string - Target string -} - -func (wm *Mention) String() string { - return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target) -} - -func (wm *Mention) asPath(conf *common.Config) string { - filename := fmt.Sprintf("%x", md5.Sum([]byte("source=" + wm.Source+ ",target=" + wm.Target))) - domain, _ := conf.FetchDomain(wm.Target) - return conf.DataPath + "/" + domain + "/" + filename + ".json" -} - -func (wm *Mention) sourceUrl() *url.URL { - url, _ := url.Parse(wm.Source) - return url -} // used as a "class" to iject dependencies, just to be able to test. Do NOT like htis. // Is there a better way? e.g. in validate, I just pass rest.Client as an arg. Not great either. @@ -45,7 +24,7 @@ type Receiver struct { Conf *common.Config } -func (recv *Receiver) Receive(wm Mention) { +func (recv *Receiver) Receive(wm mf.Mention) { log.Info().Str("Webmention", wm.String()).Msg("OK: looks valid") body, geterr := recv.RestClient.GetBody(wm.Source) @@ -58,8 +37,8 @@ func (recv *Receiver) Receive(wm Mention) { recv.processSourceBody(body, wm) } -func (recv *Receiver) deletePossibleOlderWebmention(wm Mention) { - os.Remove(wm.asPath(recv.Conf)) +func (recv *Receiver) deletePossibleOlderWebmention(wm mf.Mention) { + os.Remove(wm.AsPath(recv.Conf)) } func getHEntry(data *microformats.Data) *microformats.Microformat { @@ -72,32 +51,32 @@ func getHEntry(data *microformats.Data) *microformats.Microformat { } -func (recv *Receiver) processSourceBody(body string, wm Mention) { +func (recv *Receiver) processSourceBody(body string, wm mf.Mention) { if !strings.Contains(body, wm.Target) { log.Warn().Str("target", wm.Target).Msg("ABORT: no mention of target found in html src of source!") return } - data := microformats.Parse(strings.NewReader(body), wm.sourceUrl()) + data := microformats.Parse(strings.NewReader(body), wm.SourceUrl()) indieweb := recv.convertBodyToIndiewebData(body, wm, getHEntry(data)) recv.saveWebmentionToDisk(wm, indieweb) - log.Info().Str("file", wm.asPath(recv.Conf)).Msg("OK: Webmention processed.") + log.Info().Str("file", wm.AsPath(recv.Conf)).Msg("OK: Webmention processed.") } -func (recv *Receiver) convertBodyToIndiewebData(body string, wm Mention, hEntry *microformats.Microformat) *indiewebData { +func (recv *Receiver) convertBodyToIndiewebData(body string, wm mf.Mention, hEntry *microformats.Microformat) *mf.IndiewebData { if hEntry == nil { return recv.parseBodyAsNonIndiewebSite(body, wm) } return recv.parseBodyAsIndiewebSite(hEntry, wm) } -func (recv *Receiver) saveWebmentionToDisk(wm Mention, indieweb *indiewebData) { +func (recv *Receiver) saveWebmentionToDisk(wm mf.Mention, indieweb *mf.IndiewebData) { jsonData, jsonErr := json.Marshal(indieweb) if jsonErr != nil { log.Err(jsonErr).Msg("Unable to serialize Webmention into JSON") } - err := ioutil.WriteFile(wm.asPath(recv.Conf), jsonData, fs.ModePerm) + err := ioutil.WriteFile(wm.AsPath(recv.Conf), jsonData, fs.ModePerm) if err != nil { log.Err(err).Msg("Unable to save Webmention to disk") } @@ -105,40 +84,40 @@ func (recv *Receiver) saveWebmentionToDisk(wm Mention, indieweb *indiewebData) { // TODO I'm smelling very unstable code, apply https://golang.org/doc/effective_go#recover here? // see https://github.com/willnorris/microformats/blob/main/microformats.go -func (recv *Receiver) parseBodyAsIndiewebSite(hEntry *microformats.Microformat, wm Mention) *indiewebData { - name := mfStr(hEntry, "name") - pic := mfStr(mfProp(hEntry, "author"), "photo") - mfType := determineMfType(hEntry) +func (recv *Receiver) parseBodyAsIndiewebSite(hEntry *microformats.Microformat, wm mf.Mention) *mf.IndiewebData { + name := mf.Str(hEntry, "name") + pic := mf.Str(mf.Prop(hEntry, "author"), "photo") + mfType := mf.DetermineType(hEntry) - return &indiewebData{ + return &mf.IndiewebData{ Name: name, - Author: indiewebAuthor{ - Name: determineAuthorName(hEntry), + Author: mf.IndiewebAuthor{ + Name: mf.DetermineAuthorName(hEntry), Picture: pic, }, - Content: determineContent(hEntry), - Url: determineUrl(hEntry, wm.Source), - Published: determinePublishedDate(hEntry, recv.Conf.UtcOffset), + Content: mf.DetermineContent(hEntry), + Url: mf.DetermineUrl(hEntry, wm.Source), + Published: mf.DeterminePublishedDate(hEntry, recv.Conf.UtcOffset), Source: wm.Source, Target: wm.Target, IndiewebType: mfType, } } -func (recv *Receiver) parseBodyAsNonIndiewebSite(body string, wm Mention) *indiewebData { +func (recv *Receiver) parseBodyAsNonIndiewebSite(body string, wm mf.Mention) *mf.IndiewebData { r := regexp.MustCompile(`(.*?)<\/title>`) titleMatch := r.FindStringSubmatch(body) title := wm.Source if titleMatch != nil { title = titleMatch[1] } - return &indiewebData{ - Author: indiewebAuthor{ + return &mf.IndiewebData{ + Author: mf.IndiewebAuthor{ Name: wm.Source, }, Name: title, Content: title, - Published: publishedNow(recv.Conf.UtcOffset), + Published: mf.PublishedNow(recv.Conf.UtcOffset), Url: wm.Source, IndiewebType: "mention", Source: wm.Source, diff --git a/app/webmention/receive_test.go b/app/webmention/receive/receive_test.go similarity index 91% rename from app/webmention/receive_test.go rename to app/webmention/receive/receive_test.go index e9ea5a7..9461a04 100644 --- a/app/webmention/receive_test.go +++ b/app/webmention/receive/receive_test.go @@ -1,9 +1,10 @@ -package webmention +package receive import ( "errors" "github.com/stretchr/testify/assert" + "github.com/wgroeneveld/go-jamming/app/mf" "io/ioutil" "os" "testing" @@ -22,12 +23,12 @@ var conf = &common.Config{ func TestConvertWebmentionToPath(t *testing.T) { - wm := Mention{ + wm := mf.Mention{ Source: "https://brainbaking.com", Target: "https://jefklakscodex.com/articles", } - result := wm.asPath(conf) + result := wm.AsPath(conf) if result != "testdata/jefklakscodex.com/99be66594fdfcf482545fead8e7e4948.json" { t.Fatalf("md5 hash check failed, got " + result) } @@ -42,12 +43,12 @@ func writeSomethingTo(filename string) { func TestReceive(t *testing.T) { cases := []struct { label string - wm Mention + wm mf.Mention json string } { { label: "receive a Webmention bookmark via twitter", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-bridgy-twitter-source.html", Target: "https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag", }, @@ -55,7 +56,7 @@ func TestReceive(t *testing.T) { }, { label: "receive a brid.gy Webmention like", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-bridgy-like.html", // wrapped in a a class="u-like-of" tag Target: "https://brainbaking.com/valid-indieweb-target.html", @@ -65,7 +66,7 @@ func TestReceive(t *testing.T) { }, { label: "receive a brid.gy Webmention that has a url and photo without value", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-bridgy-source.html", Target: "https://brainbaking.com/valid-indieweb-target.html", }, @@ -73,7 +74,7 @@ func TestReceive(t *testing.T) { }, { label: "receive saves a JSON file of indieweb-metadata if all is valid", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-indieweb-source.html", Target: "https://jefklakscodex.com/articles", }, @@ -81,7 +82,7 @@ func TestReceive(t *testing.T) { }, { label: "receive saves a JSON file of indieweb-metadata with summary as content if present", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-indieweb-source-with-summary.html", Target: "https://brainbaking.com/valid-indieweb-target.html", }, @@ -89,7 +90,7 @@ func TestReceive(t *testing.T) { }, { label: "receive saves a JSON file of non-indieweb-data such as title if all is valid", - wm: Mention{ + wm: mf.Mention{ Source: "https://brainbaking.com/valid-nonindieweb-source.html", Target: "https://brainbaking.com/valid-indieweb-target.html", }, @@ -102,20 +103,20 @@ func TestReceive(t *testing.T) { os.MkdirAll("testdata/brainbaking.com", os.ModePerm) os.MkdirAll("testdata/jefklakscodex.com", os.ModePerm) defer os.RemoveAll("testdata") - now = func() time.Time { + common.Now = func() time.Time { return time.Date(2020, time.January, 1, 12, 30, 0, 0, time.UTC) } receiver := &Receiver{ Conf: conf, RestClient: &mocks.RestClientMock{ - GetBodyFunc: mocks.RelPathGetBodyFunc(t), + GetBodyFunc: mocks.RelPathGetBodyFunc(t, "../../../mocks/"), }, } receiver.Receive(tc.wm) - actualJson, _ := ioutil.ReadFile(tc.wm.asPath(conf)) + actualJson, _ := ioutil.ReadFile(tc.wm.AsPath(conf)) assert.JSONEq(t, tc.json, string(actualJson)) }) } @@ -125,11 +126,11 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi os.MkdirAll("testdata/jefklakscodex.com", os.ModePerm) defer os.RemoveAll("testdata") - wm := Mention{ + wm := mf.Mention{ Source: "https://brainbaking.com", Target: "https://jefklakscodex.com/articles", } - filename := wm.asPath(conf) + filename := wm.AsPath(conf) writeSomethingTo(filename) client := &mocks.RestClientMock{ @@ -147,17 +148,17 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi } func TestReceiveTargetThatDoesNotPointToTheSourceDoesNothing(t *testing.T) { - wm := Mention{ + wm := mf.Mention{ Source: "https://brainbaking.com/valid-indieweb-source.html", Target: "https://brainbaking.com/valid-indieweb-source.html", } - filename := wm.asPath(conf) + filename := wm.AsPath(conf) writeSomethingTo(filename) receiver := &Receiver{ Conf: conf, RestClient: &mocks.RestClientMock{ - GetBodyFunc: mocks.RelPathGetBodyFunc(t), + GetBodyFunc: mocks.RelPathGetBodyFunc(t, "../../../mocks/"), }, } @@ -169,7 +170,7 @@ func TestProcessSourceBodyAbortsIfNoMentionOfTargetFoundInSourceHtml(t *testing. os.MkdirAll("testdata/jefklakscodex.com", os.ModePerm) defer os.RemoveAll("testdata") - wm := Mention{ + wm := mf.Mention{ Source: "https://brainbaking.com", Target: "https://jefklakscodex.com/articles", } @@ -178,6 +179,6 @@ func TestProcessSourceBodyAbortsIfNoMentionOfTargetFoundInSourceHtml(t *testing. } receiver.processSourceBody("<html>my nice body</html>", wm) - assert.NoFileExists(t, wm.asPath(conf)) + assert.NoFileExists(t, wm.AsPath(conf)) } diff --git a/app/webmention/send/send.go b/app/webmention/send/send.go new file mode 100644 index 0000000..46d64be --- /dev/null +++ b/app/webmention/send/send.go @@ -0,0 +1,10 @@ +package send + +import ( + "github.com/wgroeneveld/go-jamming/app/mf" + "github.com/wgroeneveld/go-jamming/app/pingback/send" +) + +func mention() { + send.SendPingbackToEndpoint("endpoint", mf.Mention{}) +} diff --git a/common/time.go b/common/time.go new file mode 100644 index 0000000..69bfba4 --- /dev/null +++ b/common/time.go @@ -0,0 +1,6 @@ +package common + +import "time" + +// I know it's public. Not sure how to handle this in tests, package-independent +var Now = time.Now diff --git a/mocks/restclient.go b/mocks/restclient.go index 8d02693..87aeaea 100644 --- a/mocks/restclient.go +++ b/mocks/restclient.go @@ -22,11 +22,11 @@ func (m *RestClientMock) GetBody(url string) (string, error) { return m.GetBodyFunc(url) } -func RelPathGetBodyFunc(t *testing.T) func(string) (string, error) { +func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (string, error) { return func(url string) (string, error) { // url: https://brainbaking.com/something-something.html // want: ../../mocks/something-something.html - mockfile := "../../mocks/" + strings.ReplaceAll(url, "https://brainbaking.com/", "") + mockfile := relPath + strings.ReplaceAll(url, "https://brainbaking.com/", "") html, err := ioutil.ReadFile(mockfile) if err != nil { t.Error(err)