From 714b90d594948355812105ded926408b415134b8 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Sun, 11 Apr 2021 09:50:27 +0200 Subject: [PATCH] wm/pingback sending, concurrent impl, e2e test --- app/mf/mention.go | 14 ++++- app/mf/microformats.go | 9 +++ app/pingback/send/send.go | 5 +- app/webmention/recv/receive.go | 13 +---- app/webmention/recv/receive_test.go | 5 +- app/webmention/send/discoverer.go | 48 +++++++++++++++ app/webmention/send/discoverer_test.go | 78 +++++++++++++++++++++++++ app/webmention/send/send.go | 60 ++++++++++++++++--- app/webmention/send/send_test.go | 81 ++++++++++++++++++++++++++ common/collections.go | 9 +++ common/slices.go | 10 ---- mocks/restclient.go | 39 ++++++++++--- rest/client.go | 48 +++++++++++---- rest/utils.go | 8 +++ 14 files changed, 374 insertions(+), 53 deletions(-) create mode 100644 app/webmention/send/discoverer.go create mode 100644 app/webmention/send/discoverer_test.go create mode 100644 app/webmention/send/send_test.go delete mode 100644 common/slices.go diff --git a/app/mf/mention.go b/app/mf/mention.go index 6c91681..551cd95 100644 --- a/app/mf/mention.go +++ b/app/mf/mention.go @@ -7,22 +7,30 @@ import ( "net/url" ) +// this should be passed along as a value object, not as a pointer type Mention struct { Source string Target string } -func (wm *Mention) String() string { +func (wm Mention) AsFormValues() url.Values { + values := url.Values{} + values.Add("source", wm.Source) + values.Add("target", wm.Target) + return values +} + +func (wm Mention) String() string { return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target) } -func (wm *Mention) AsPath(conf *common.Config) string { +func (wm Mention) AsPath(conf *common.Config) string { filename := fmt.Sprintf("%x", md5.Sum([]byte("source="+wm.Source+",target="+wm.Target))) domain, _ := conf.FetchDomain(wm.Target) return conf.DataPath + "/" + domain + "/" + filename + ".json" } -func (wm *Mention) SourceUrl() *url.URL { +func (wm Mention) SourceUrl() *url.URL { url, _ := url.Parse(wm.Source) return url } diff --git a/app/mf/microformats.go b/app/mf/microformats.go index 9042197..1eb7a2b 100644 --- a/app/mf/microformats.go +++ b/app/mf/microformats.go @@ -72,6 +72,15 @@ func Map(mf *microformats.Microformat, key string) map[string]string { return mapVal } +func HEntry(data *microformats.Data) *microformats.Microformat { + for _, itm := range data.Items { + if common.Includes(itm.Type, "h-entry") { + return itm + } + } + return nil +} + func Prop(mf *microformats.Microformat, key string) *microformats.Microformat { val := mf.Properties[key] if len(val) == 0 { diff --git a/app/pingback/send/send.go b/app/pingback/send/send.go index 4e80682..bad6bd0 100644 --- a/app/pingback/send/send.go +++ b/app/pingback/send/send.go @@ -44,7 +44,8 @@ type Sender struct { func (sender *Sender) SendPingbackToEndpoint(endpoint string, mention mf.Mention) { err := sender.RestClient.Post(endpoint, "text/xml", body.fill(mention)) if err != nil { - log.Err(err).Str("wm", mention.String()).Msg("Unable to send pingback") + log.Err(err).Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Unable to send pingback") + return } - log.Info().Str("wm", mention.String()).Msg("Pingback sent") + log.Info().Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Pingback sent") } diff --git a/app/webmention/recv/receive.go b/app/webmention/recv/receive.go index 356b508..c29e150 100644 --- a/app/webmention/recv/receive.go +++ b/app/webmention/recv/receive.go @@ -24,7 +24,7 @@ type Receiver struct { func (recv *Receiver) Receive(wm mf.Mention) { log.Info().Str("Webmention", wm.String()).Msg("OK: looks valid") - body, geterr := recv.RestClient.GetBody(wm.Source) + _, body, geterr := recv.RestClient.GetBody(wm.Source) if geterr != nil { log.Warn().Str("source", wm.Source).Msg(" ABORT: invalid url") @@ -39,15 +39,6 @@ func (recv *Receiver) deletePossibleOlderWebmention(wm mf.Mention) { os.Remove(wm.AsPath(recv.Conf)) } -func getHEntry(data *microformats.Data) *microformats.Microformat { - for _, itm := range data.Items { - if common.Includes(itm.Type, "h-entry") { - return itm - } - } - return nil -} - func (recv *Receiver) processSourceBody(body string, wm mf.Mention) { if !strings.Contains(body, wm.Target) { log.Warn().Str("target", wm.Target).Msg("ABORT: no mention of target found in html src of source!") @@ -55,7 +46,7 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) { } data := microformats.Parse(strings.NewReader(body), wm.SourceUrl()) - indieweb := recv.convertBodyToIndiewebData(body, wm, getHEntry(data)) + indieweb := recv.convertBodyToIndiewebData(body, wm, mf.HEntry(data)) recv.saveWebmentionToDisk(wm, indieweb) log.Info().Str("file", wm.AsPath(recv.Conf)).Msg("OK: Webmention processed.") diff --git a/app/webmention/recv/receive_test.go b/app/webmention/recv/receive_test.go index aa2e338..c1c0053 100644 --- a/app/webmention/recv/receive_test.go +++ b/app/webmention/recv/receive_test.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/stretchr/testify/assert" "io/ioutil" + "net/http" "os" "testing" "time" @@ -132,8 +133,8 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi writeSomethingTo(filename) client := &mocks.RestClientMock{ - GetBodyFunc: func(url string) (string, error) { - return "", errors.New("whoops") + GetBodyFunc: func(url string) (http.Header, string, error) { + return nil, "", errors.New("whoops") }, } receiver := &Receiver{ diff --git a/app/webmention/send/discoverer.go b/app/webmention/send/discoverer.go new file mode 100644 index 0000000..1dc3ae6 --- /dev/null +++ b/app/webmention/send/discoverer.go @@ -0,0 +1,48 @@ +package send + +import ( + "brainbaking.com/go-jamming/rest" + "github.com/rs/zerolog/log" + "strings" + "willnorris.com/go/microformats" +) + +const ( + TypeWebmention string = "webmention" + TypeUnknown string = "unknown" + TypePingback string = "pingback" +) + +func (sndr *Sender) discover(target string) (link string, mentionType string) { + mentionType = TypeUnknown + header, body, err := sndr.RestClient.GetBody(target) + if err != nil { + log.Warn().Str("target", target).Msg("Failed to discover possible endpoint, aborting send") + return + } + + if strings.Contains(header.Get("link"), TypeWebmention) { + return buildWebmentionHeaderLink(header.Get("link")), TypeWebmention + } + if header.Get("X-Pingback") != "" { + return header.Get("X-Pingback"), TypePingback + } + + // this also complies with w3.org regulations: relative endpoint could be possible + format := microformats.Parse(strings.NewReader(body), rest.BaseUrlOf(target)) + if len(format.Rels[TypeWebmention]) > 0 { + mentionType = TypeWebmention + link = format.Rels[TypeWebmention][0] + } else if len(format.Rels[TypePingback]) > 0 { + mentionType = TypePingback + link = format.Rels[TypePingback][0] + } + + return +} + +func buildWebmentionHeaderLink(link string) string { + // e.g. Link: ; rel="webmention" + raw := strings.Split(link, ";")[0][1:] + return raw[:len(raw)-1] +} diff --git a/app/webmention/send/discoverer_test.go b/app/webmention/send/discoverer_test.go new file mode 100644 index 0000000..4df0d71 --- /dev/null +++ b/app/webmention/send/discoverer_test.go @@ -0,0 +1,78 @@ +package send + +import ( + "brainbaking.com/go-jamming/mocks" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDiscover(t *testing.T) { + var sender = &Sender{ + RestClient: &mocks.RestClientMock{ + GetBodyFunc: mocks.RelPathGetBodyFunc(t, "../../../mocks/"), + }, + } + + cases := []struct { + label string + url string + expectedLink string + expectedType string + }{ + { + "discover 'unknown' if no link is present", + "https://brainbaking.com/link-discover-test-none.html", + "", + TypeUnknown, + }, + { + "prefer webmentions over pingbacks if both links are present", + "https://brainbaking.com/link-discover-bothtypes.html", + "http://aaronpk.example/webmention-endpoint", + TypeWebmention, + }, + { + "pingbacks: discover link if present in header", + "https://brainbaking.com/pingback-discover-test.html", + "http://aaronpk.example/pingback-endpoint", + TypePingback, + }, + { + "pingbacks: discover link if sole entry somewhere in html", + "https://brainbaking.com/pingback-discover-test-single.html", + "http://aaronpk.example/pingback-endpoint-body", + TypePingback, + }, + { + "pingbacks: use link in header if multiple present in html", + "https://brainbaking.com/pingback-discover-test-multiple.html", + "http://aaronpk.example/pingback-endpoint-header", + TypePingback, + }, + { + "webmentions: discover link if present in header", + "https://brainbaking.com/link-discover-test.html", + "http://aaronpk.example/webmention-endpoint", + TypeWebmention, + }, + { + "webmentions: discover link if sole entry somewhere in html", + "https://brainbaking.com/link-discover-test-single.html", + "http://aaronpk.example/webmention-endpoint-body", + TypeWebmention, + }, + { + "webmentions: use link in header if multiple present in html", + "https://brainbaking.com/link-discover-test-multiple.html", + "http://aaronpk.example/webmention-endpoint-header", + TypeWebmention, + }, + } + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + link, mentionType := sender.discover(tc.url) + assert.Equal(t, tc.expectedLink, link) + assert.Equal(t, tc.expectedType, mentionType) + }) + } +} diff --git a/app/webmention/send/send.go b/app/webmention/send/send.go index d5d0d75..c6efc54 100644 --- a/app/webmention/send/send.go +++ b/app/webmention/send/send.go @@ -6,6 +6,7 @@ import ( "brainbaking.com/go-jamming/common" "brainbaking.com/go-jamming/rest" "github.com/rs/zerolog/log" + "sync" "time" ) @@ -16,22 +17,65 @@ type Sender struct { func (snder *Sender) Send(domain string, since string) { log.Info().Str("domain", domain).Str("since", since).Msg(` OK: someone wants to send mentions`) - feed, err := snder.RestClient.GetBody("https://" + domain + "/index.xml") + _, feed, err := snder.RestClient.GetBody("https://" + domain + "/index.xml") if err != nil { log.Err(err).Str("domain", domain).Msg("Unable to retrieve RSS feed, aborting send") return } snder.parseRssFeed(feed, common.IsoToTime(since)) + log.Info().Str("domain", domain).Str("since", since).Msg(` OK: sending done.`) } func (snder *Sender) parseRssFeed(feed string, since time.Time) { - -} - -func mention() { - pingbackSender := &send.Sender{ - RestClient: nil, + items, err := snder.Collect(feed, since) + if err != nil { + log.Err(err).Msg("Unable to parse RSS fed, aborting send") + return } - pingbackSender.SendPingbackToEndpoint("endpoint", mf.Mention{}) + + var wg sync.WaitGroup + for _, item := range items { + for _, href := range item.hrefs { + mention := mf.Mention{ + // SOURCE is own domain this time, TARGET = outbound + Source: item.link, + Target: href, + } + + wg.Add(1) + go func() { + defer wg.Done() + snder.sendMention(mention) + }() + } + } + wg.Wait() +} + +var mentionFuncs = map[string]func(snder *Sender, mention mf.Mention, endpoint string){ + TypeUnknown: func(snder *Sender, mention mf.Mention, endpoint string) {}, + TypeWebmention: sendMentionAsWebmention, + TypePingback: sendMentionAsPingback, +} + +func (snder *Sender) sendMention(mention mf.Mention) { + endpoint, mentionType := snder.discover(mention.Target) + mentionFuncs[mentionType](snder, mention, endpoint) +} + +func sendMentionAsWebmention(snder *Sender, mention mf.Mention, endpoint string) { + err := snder.RestClient.PostForm(endpoint, mention.AsFormValues()) + if err != nil { + log.Err(err).Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Webmention send failed") + return + } + log.Info().Str("endpoint", endpoint).Str("wm", mention.String()).Msg("OK: webmention sent.") +} + +func sendMentionAsPingback(snder *Sender, mention mf.Mention, endpoint string) { + pingbackSender := &send.Sender{ + RestClient: snder.RestClient, + } + pingbackSender.SendPingbackToEndpoint(endpoint, mention) } diff --git a/app/webmention/send/send_test.go b/app/webmention/send/send_test.go new file mode 100644 index 0000000..e1a2880 --- /dev/null +++ b/app/webmention/send/send_test.go @@ -0,0 +1,81 @@ +package send + +import ( + "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/common" + "brainbaking.com/go-jamming/mocks" + "github.com/stretchr/testify/assert" + "net/url" + "sync" + "testing" +) + +func TestSendMentionAsWebmention(t *testing.T) { + passedFormValues := url.Values{} + snder := Sender{ + RestClient: &mocks.RestClientMock{ + PostFormFunc: func(endpoint string, formValues url.Values) error { + passedFormValues = formValues + return nil + }, + }, + } + + sendMentionAsWebmention(&snder, mf.Mention{ + Source: "mysource", + Target: "mytarget", + }, "someendpoint") + + assert.Equal(t, "mysource", passedFormValues.Get("source")) + assert.Equal(t, "mytarget", passedFormValues.Get("target")) +} + +func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) { + posted := map[string]interface{}{} + var lock = sync.RWMutex{} + + snder := Sender{ + Conf: common.Configure(), + RestClient: &mocks.RestClientMock{ + GetBodyFunc: mocks.RelPathGetBodyFunc(t, "./../../../mocks/"), + PostFunc: func(url string, contentType string, body string) error { + lock.RLock() + defer lock.RUnlock() + posted[url] = body + return nil + }, + PostFormFunc: func(endpoint string, formValues url.Values) error { + lock.RLock() + defer lock.RUnlock() + posted[endpoint] = formValues + return nil + }, + }, + } + + snder.Send("brainbaking.com", "2021-03-16T16:00:00.000Z") + assert.Equal(t, 3, len(posted)) + + wmPost1 := posted["http://aaronpk.example/webmention-endpoint-header"].(url.Values) + assert.Equal(t, "https://brainbaking.com/notes/2021/03/16h17m07s14/", wmPost1.Get("source")) + assert.Equal(t, "https://brainbaking.com/link-discover-test-multiple.html", wmPost1.Get("target")) + + wmPost2 := posted["http://aaronpk.example/pingback-endpoint-body"].(string) + expectedPost2 := ` + + pingback.ping + + + https://brainbaking.com/notes/2021/03/16h17m07s14/ + + + https://brainbaking.com/pingback-discover-test-single.html + + +` + assert.Equal(t, expectedPost2, wmPost2) + + wmPost3 := posted["http://aaronpk.example/webmention-endpoint-body"].(url.Values) + assert.Equal(t, "https://brainbaking.com/notes/2021/03/16h17m07s14/", wmPost3.Get("source")) + assert.Equal(t, "https://brainbaking.com/link-discover-test-single.html", wmPost3.Get("target")) +} diff --git a/common/collections.go b/common/collections.go index 320d3b9..4dffc42 100644 --- a/common/collections.go +++ b/common/collections.go @@ -38,3 +38,12 @@ func (set *Set) Keys() []string { } return keys } + +func Includes(slice []string, elem string) bool { + for _, el := range slice { + if el == elem { + return true + } + } + return false +} diff --git a/common/slices.go b/common/slices.go deleted file mode 100644 index bde04ff..0000000 --- a/common/slices.go +++ /dev/null @@ -1,10 +0,0 @@ -package common - -func Includes(slice []string, elem string) bool { - for _, el := range slice { - if el == elem { - return true - } - } - return false -} diff --git a/mocks/restclient.go b/mocks/restclient.go index b877724..517c67d 100644 --- a/mocks/restclient.go +++ b/mocks/restclient.go @@ -1,33 +1,50 @@ package mocks import ( + "encoding/json" + "fmt" "io/ioutil" "net/http" + "net/url" "strings" "testing" ) // neat trick! https://medium.com/@matryer/meet-moq-easily-mock-interfaces-in-go-476444187d10 type RestClientMock struct { - GetFunc func(string) (*http.Response, error) - GetBodyFunc func(string) (string, error) - PostFunc func(string, string, string) error + GetFunc func(string) (*http.Response, error) + GetBodyFunc func(string) (http.Header, string, error) + PostFunc func(string, string, string) error + PostFormFunc func(string, url.Values) error } // although these are still requied to match the rest.Client interface. func (m *RestClientMock) Get(url string) (*http.Response, error) { return m.GetFunc(url) } -func (m *RestClientMock) GetBody(url string) (string, error) { +func (m *RestClientMock) GetBody(url string) (http.Header, string, error) { return m.GetBodyFunc(url) } +func (client *RestClientMock) PostForm(url string, formData url.Values) error { + return client.PostFormFunc(url, formData) +} + func (m *RestClientMock) Post(url string, contentType string, body string) error { return m.PostFunc(url, contentType, body) } -func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (string, error) { - return func(url string) (string, error) { +func toHttpHeader(header map[string]interface{}) http.Header { + httpHeader := http.Header{} + for key, value := range header { + httpHeader.Add(key, value.(string)) + } + return httpHeader +} + +func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (http.Header, string, error) { + return func(url string) (http.Header, string, error) { + fmt.Println(" - GET call at " + url) // url: https://brainbaking.com/something-something.html // want: ../../mocks/something-something.html mockfile := relPath + strings.ReplaceAll(url, "https://brainbaking.com/", "") @@ -35,7 +52,15 @@ func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (string, erro if err != nil { t.Error(err) } - return string(html), nil + + headerData, headerFileErr := ioutil.ReadFile(strings.ReplaceAll(mockfile, ".html", "-headers.json")) + if headerFileErr != nil { + return http.Header{}, string(html), nil + } + headerJson := map[string]interface{}{} + json.Unmarshal(headerData, &headerJson) + + return toHttpHeader(headerJson), string(html), nil } } diff --git a/rest/client.go b/rest/client.go index a2703d3..fdf7540 100644 --- a/rest/client.go +++ b/rest/client.go @@ -4,46 +4,74 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strings" ) type Client interface { Get(url string) (*http.Response, error) Post(url string, contentType string, body string) error - GetBody(url string) (string, error) + GetBody(url string) (http.Header, string, error) + PostForm(url string, formData url.Values) error } type HttpClient struct { } -func (client *HttpClient) Post(url string, contenType string, body string) error { - _, err := http.Post(url, contenType, strings.NewReader(body)) +func (client *HttpClient) PostForm(url string, formData url.Values) error { + resp, err := http.PostForm(url, formData) if err != nil { return err } + if !isStatusOk(resp) { + return fmt.Errorf("POST Form to %s: Status code is not OK (%d)", url, resp.StatusCode) + } + return nil +} + +func (client *HttpClient) Post(url string, contenType string, body string) error { + resp, err := http.Post(url, contenType, strings.NewReader(body)) + if err != nil { + return err + } + if !isStatusOk(resp) { + return fmt.Errorf("POST to %s: Status code is not OK (%d)", url, resp.StatusCode) + } return nil } // something like this? https://freshman.tech/snippets/go/http-response-to-string/ -func (client *HttpClient) GetBody(url string) (string, error) { - resp, geterr := http.Get(url) +func (client *HttpClient) GetBody(url string) (http.Header, string, error) { + resp, geterr := client.Get(url) if geterr != nil { - return "", geterr + return nil, "", geterr } - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return "", fmt.Errorf("Status code for %s is not OK (%d)", url, resp.StatusCode) + body, err := ReadBodyFromResponse(resp) + if err != nil { + return nil, "", err + } + + return resp.Header, body, nil +} + +func ReadBodyFromResponse(resp *http.Response) (string, error) { + if !isStatusOk(resp) { + return "", fmt.Errorf("Status code is not OK (%d)", resp.StatusCode) } - defer resp.Body.Close() body, readerr := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() if readerr != nil { return "", readerr } - return string(body), nil } +func isStatusOk(resp *http.Response) bool { + return resp.StatusCode >= 200 && resp.StatusCode <= 299 +} + func (client *HttpClient) Get(url string) (*http.Response, error) { return http.Get(url) } diff --git a/rest/utils.go b/rest/utils.go index e24336b..637e445 100644 --- a/rest/utils.go +++ b/rest/utils.go @@ -2,6 +2,7 @@ package rest import ( "net/http" + "net/url" ) func BadRequest(w http.ResponseWriter) { @@ -12,3 +13,10 @@ func Accept(w http.ResponseWriter) { w.WriteHeader(202) w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!")) } + +// assumes the URL is well-formed. +func BaseUrlOf(link string) *url.URL { + obj, _ := url.Parse(link) + baseUrl, _ := url.Parse(obj.Scheme + "://" + obj.Host) + return baseUrl +}