wgroeneveld
/
go-jamming
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

wm/pingback sending, concurrent impl, e2e test

This commit is contained in:
Wouter Groeneveld 2021-04-11 09:50:27 +02:00
parent d9ded09383
commit 714b90d594
14 changed files with 374 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/mf/mention.go
View File

@ -7,22 +7,30 @@ import (
"net/url"
)
// this should be passed along as a value object, not as a pointer
type Mention struct {
Source string
Target string
}
func (wm *Mention) String() string {
func (wm Mention) AsFormValues() url.Values {
values := url.Values{}
values.Add("source", wm.Source)
values.Add("target", wm.Target)
return values
}
func (wm Mention) String() string {
return fmt.Sprintf("source: %s, target: %s", wm.Source, wm.Target)
}
func (wm *Mention) AsPath(conf *common.Config) string {
func (wm Mention) AsPath(conf *common.Config) string {
filename := fmt.Sprintf("%x", md5.Sum([]byte("source="+wm.Source+",target="+wm.Target)))
domain, _ := conf.FetchDomain(wm.Target)
return conf.DataPath + "/" + domain + "/" + filename + ".json"
}
func (wm *Mention) SourceUrl() *url.URL {
func (wm Mention) SourceUrl() *url.URL {
url, _ := url.Parse(wm.Source)
return url
}

9
app/mf/microformats.go
View File

@ -72,6 +72,15 @@ func Map(mf *microformats.Microformat, key string) map[string]string {
return mapVal
}
func HEntry(data *microformats.Data) *microformats.Microformat {
for _, itm := range data.Items {
if common.Includes(itm.Type, "h-entry") {
return itm
}
}
return nil
}
func Prop(mf *microformats.Microformat, key string) *microformats.Microformat {
val := mf.Properties[key]
if len(val) == 0 {

5
app/pingback/send/send.go
View File

@ -44,7 +44,8 @@ type Sender struct {
func (sender *Sender) SendPingbackToEndpoint(endpoint string, mention mf.Mention) {
err := sender.RestClient.Post(endpoint, "text/xml", body.fill(mention))
if err != nil {
log.Err(err).Str("wm", mention.String()).Msg("Unable to send pingback")
log.Err(err).Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Unable to send pingback")
return
}
log.Info().Str("wm", mention.String()).Msg("Pingback sent")
log.Info().Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Pingback sent")
}

13
app/webmention/recv/receive.go
View File

@ -24,7 +24,7 @@ type Receiver struct {
func (recv *Receiver) Receive(wm mf.Mention) {
log.Info().Str("Webmention", wm.String()).Msg("OK: looks valid")
body, geterr := recv.RestClient.GetBody(wm.Source)
_, body, geterr := recv.RestClient.GetBody(wm.Source)
if geterr != nil {
log.Warn().Str("source", wm.Source).Msg(" ABORT: invalid url")
@ -39,15 +39,6 @@ func (recv *Receiver) deletePossibleOlderWebmention(wm mf.Mention) {
os.Remove(wm.AsPath(recv.Conf))
}
func getHEntry(data *microformats.Data) *microformats.Microformat {
for _, itm := range data.Items {
if common.Includes(itm.Type, "h-entry") {
return itm
}
}
return nil
}
func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {
if !strings.Contains(body, wm.Target) {
log.Warn().Str("target", wm.Target).Msg("ABORT: no mention of target found in html src of source!")
@ -55,7 +46,7 @@ func (recv *Receiver) processSourceBody(body string, wm mf.Mention) {
}
data := microformats.Parse(strings.NewReader(body), wm.SourceUrl())
indieweb := recv.convertBodyToIndiewebData(body, wm, getHEntry(data))
indieweb := recv.convertBodyToIndiewebData(body, wm, mf.HEntry(data))
recv.saveWebmentionToDisk(wm, indieweb)
log.Info().Str("file", wm.AsPath(recv.Conf)).Msg("OK: Webmention processed.")

5
app/webmention/recv/receive_test.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"github.com/stretchr/testify/assert"
"io/ioutil"
"net/http"
"os"
"testing"
"time"
@ -132,8 +133,8 @@ func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testi
writeSomethingTo(filename)
client := &mocks.RestClientMock{
GetBodyFunc: func(url string) (string, error) {
return "", errors.New("whoops")
GetBodyFunc: func(url string) (http.Header, string, error) {
return nil, "", errors.New("whoops")
},
}
receiver := &Receiver{

48
app/webmention/send/discoverer.go Normal file
View File

@ -0,0 +1,48 @@
package send
import (
"brainbaking.com/go-jamming/rest"
"github.com/rs/zerolog/log"
"strings"
"willnorris.com/go/microformats"
)
const (
TypeWebmention string = "webmention"
TypeUnknown string = "unknown"
TypePingback string = "pingback"
)
func (sndr *Sender) discover(target string) (link string, mentionType string) {
mentionType = TypeUnknown
header, body, err := sndr.RestClient.GetBody(target)
if err != nil {
log.Warn().Str("target", target).Msg("Failed to discover possible endpoint, aborting send")
return
}
if strings.Contains(header.Get("link"), TypeWebmention) {
return buildWebmentionHeaderLink(header.Get("link")), TypeWebmention
}
if header.Get("X-Pingback") != "" {
return header.Get("X-Pingback"), TypePingback
}
// this also complies with w3.org regulations: relative endpoint could be possible
format := microformats.Parse(strings.NewReader(body), rest.BaseUrlOf(target))
if len(format.Rels[TypeWebmention]) > 0 {
mentionType = TypeWebmention
link = format.Rels[TypeWebmention][0]
} else if len(format.Rels[TypePingback]) > 0 {
mentionType = TypePingback
link = format.Rels[TypePingback][0]
}
return
}
func buildWebmentionHeaderLink(link string) string {
// e.g. Link: <http://aaronpk.example/webmention-endpoint>; rel="webmention"
raw := strings.Split(link, ";")[0][1:]
return raw[:len(raw)-1]
}

78
app/webmention/send/discoverer_test.go Normal file
View File

@ -0,0 +1,78 @@
package send
import (
"brainbaking.com/go-jamming/mocks"
"github.com/stretchr/testify/assert"
"testing"
)
func TestDiscover(t *testing.T) {
var sender = &Sender{
RestClient: &mocks.RestClientMock{
GetBodyFunc: mocks.RelPathGetBodyFunc(t, "../../../mocks/"),
},
}
cases := []struct {
label string
url string
expectedLink string
expectedType string
}{
{
"discover 'unknown' if no link is present",
"https://brainbaking.com/link-discover-test-none.html",
"",
TypeUnknown,
},
{
"prefer webmentions over pingbacks if both links are present",
"https://brainbaking.com/link-discover-bothtypes.html",
"http://aaronpk.example/webmention-endpoint",
TypeWebmention,
},
{
"pingbacks: discover link if present in header",
"https://brainbaking.com/pingback-discover-test.html",
"http://aaronpk.example/pingback-endpoint",
TypePingback,
},
{
"pingbacks: discover link if sole entry somewhere in html",
"https://brainbaking.com/pingback-discover-test-single.html",
"http://aaronpk.example/pingback-endpoint-body",
TypePingback,
},
{
"pingbacks: use link in header if multiple present in html",
"https://brainbaking.com/pingback-discover-test-multiple.html",
"http://aaronpk.example/pingback-endpoint-header",
TypePingback,
},
{
"webmentions: discover link if present in header",
"https://brainbaking.com/link-discover-test.html",
"http://aaronpk.example/webmention-endpoint",
TypeWebmention,
},
{
"webmentions: discover link if sole entry somewhere in html",
"https://brainbaking.com/link-discover-test-single.html",
"http://aaronpk.example/webmention-endpoint-body",
TypeWebmention,
},
{
"webmentions: use link in header if multiple present in html",
"https://brainbaking.com/link-discover-test-multiple.html",
"http://aaronpk.example/webmention-endpoint-header",
TypeWebmention,
},
}
for _, tc := range cases {
t.Run(tc.label, func(t *testing.T) {
link, mentionType := sender.discover(tc.url)
assert.Equal(t, tc.expectedLink, link)
assert.Equal(t, tc.expectedType, mentionType)
})
}
}

60
app/webmention/send/send.go
View File

@ -6,6 +6,7 @@ import (
"brainbaking.com/go-jamming/common"
"brainbaking.com/go-jamming/rest"
"github.com/rs/zerolog/log"
"sync"
"time"
)
@ -16,22 +17,65 @@ type Sender struct {
func (snder *Sender) Send(domain string, since string) {
log.Info().Str("domain", domain).Str("since", since).Msg(` OK: someone wants to send mentions`)
feed, err := snder.RestClient.GetBody("https://" + domain + "/index.xml")
_, feed, err := snder.RestClient.GetBody("https://" + domain + "/index.xml")
if err != nil {
log.Err(err).Str("domain", domain).Msg("Unable to retrieve RSS feed, aborting send")
return
}
snder.parseRssFeed(feed, common.IsoToTime(since))
log.Info().Str("domain", domain).Str("since", since).Msg(` OK: sending done.`)
}
func (snder *Sender) parseRssFeed(feed string, since time.Time) {
}
func mention() {
pingbackSender := &send.Sender{
RestClient: nil,
items, err := snder.Collect(feed, since)
if err != nil {
log.Err(err).Msg("Unable to parse RSS fed, aborting send")
return
}
pingbackSender.SendPingbackToEndpoint("endpoint", mf.Mention{})
var wg sync.WaitGroup
for _, item := range items {
for _, href := range item.hrefs {
mention := mf.Mention{
// SOURCE is own domain this time, TARGET = outbound
Source: item.link,
Target: href,
}
wg.Add(1)
go func() {
defer wg.Done()
snder.sendMention(mention)
}()
}
}
wg.Wait()
}
var mentionFuncs = map[string]func(snder *Sender, mention mf.Mention, endpoint string){
TypeUnknown: func(snder *Sender, mention mf.Mention, endpoint string) {},
TypeWebmention: sendMentionAsWebmention,
TypePingback: sendMentionAsPingback,
}
func (snder *Sender) sendMention(mention mf.Mention) {
endpoint, mentionType := snder.discover(mention.Target)
mentionFuncs[mentionType](snder, mention, endpoint)
}
func sendMentionAsWebmention(snder *Sender, mention mf.Mention, endpoint string) {
err := snder.RestClient.PostForm(endpoint, mention.AsFormValues())
if err != nil {
log.Err(err).Str("endpoint", endpoint).Str("wm", mention.String()).Msg("Webmention send failed")
return
}
log.Info().Str("endpoint", endpoint).Str("wm", mention.String()).Msg("OK: webmention sent.")
}
func sendMentionAsPingback(snder *Sender, mention mf.Mention, endpoint string) {
pingbackSender := &send.Sender{
RestClient: snder.RestClient,
}
pingbackSender.SendPingbackToEndpoint(endpoint, mention)
}

81
app/webmention/send/send_test.go Normal file
View File

@ -0,0 +1,81 @@
package send
import (
"brainbaking.com/go-jamming/app/mf"
"brainbaking.com/go-jamming/common"
"brainbaking.com/go-jamming/mocks"
"github.com/stretchr/testify/assert"
"net/url"
"sync"
"testing"
)
func TestSendMentionAsWebmention(t *testing.T) {
passedFormValues := url.Values{}
snder := Sender{
RestClient: &mocks.RestClientMock{
PostFormFunc: func(endpoint string, formValues url.Values) error {
passedFormValues = formValues
return nil
},
},
}
sendMentionAsWebmention(&snder, mf.Mention{
Source: "mysource",
Target: "mytarget",
}, "someendpoint")
assert.Equal(t, "mysource", passedFormValues.Get("source"))
assert.Equal(t, "mytarget", passedFormValues.Get("target"))
}
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
posted := map[string]interface{}{}
var lock = sync.RWMutex{}
snder := Sender{
Conf: common.Configure(),
RestClient: &mocks.RestClientMock{
GetBodyFunc: mocks.RelPathGetBodyFunc(t, "./../../../mocks/"),
PostFunc: func(url string, contentType string, body string) error {
lock.RLock()
defer lock.RUnlock()
posted[url] = body
return nil
},
PostFormFunc: func(endpoint string, formValues url.Values) error {
lock.RLock()
defer lock.RUnlock()
posted[endpoint] = formValues
return nil
},
},
}
snder.Send("brainbaking.com", "2021-03-16T16:00:00.000Z")
assert.Equal(t, 3, len(posted))
wmPost1 := posted["http://aaronpk.example/webmention-endpoint-header"].(url.Values)
assert.Equal(t, "https://brainbaking.com/notes/2021/03/16h17m07s14/", wmPost1.Get("source"))
assert.Equal(t, "https://brainbaking.com/link-discover-test-multiple.html", wmPost1.Get("target"))
wmPost2 := posted["http://aaronpk.example/pingback-endpoint-body"].(string)
expectedPost2 := `<?xml version="1.0" encoding="UTF-8"?>
<methodCall>
<methodName>pingback.ping</methodName>
<params>
<param>
<value><string>https://brainbaking.com/notes/2021/03/16h17m07s14/</string></value>
</param>
<param>
<value><string>https://brainbaking.com/pingback-discover-test-single.html</string></value>
</param>
</params>
</methodCall>`
assert.Equal(t, expectedPost2, wmPost2)
wmPost3 := posted["http://aaronpk.example/webmention-endpoint-body"].(url.Values)
assert.Equal(t, "https://brainbaking.com/notes/2021/03/16h17m07s14/", wmPost3.Get("source"))
assert.Equal(t, "https://brainbaking.com/link-discover-test-single.html", wmPost3.Get("target"))
}

9
common/collections.go
View File

@ -38,3 +38,12 @@ func (set *Set) Keys() []string {
}
return keys
}
func Includes(slice []string, elem string) bool {
for _, el := range slice {
if el == elem {
return true
}
}
return false
}

10
common/slices.go
View File

@ -1,10 +0,0 @@
package common
func Includes(slice []string, elem string) bool {
for _, el := range slice {
if el == elem {
return true
}
}
return false
}

39
mocks/restclient.go
View File

@ -1,33 +1,50 @@
package mocks
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
)
// neat trick! https://medium.com/@matryer/meet-moq-easily-mock-interfaces-in-go-476444187d10
type RestClientMock struct {
GetFunc func(string) (*http.Response, error)
GetBodyFunc func(string) (string, error)
PostFunc func(string, string, string) error
GetFunc func(string) (*http.Response, error)
GetBodyFunc func(string) (http.Header, string, error)
PostFunc func(string, string, string) error
PostFormFunc func(string, url.Values) error
}
// although these are still requied to match the rest.Client interface.
func (m *RestClientMock) Get(url string) (*http.Response, error) {
return m.GetFunc(url)
}
func (m *RestClientMock) GetBody(url string) (string, error) {
func (m *RestClientMock) GetBody(url string) (http.Header, string, error) {
return m.GetBodyFunc(url)
}
func (client *RestClientMock) PostForm(url string, formData url.Values) error {
return client.PostFormFunc(url, formData)
}
func (m *RestClientMock) Post(url string, contentType string, body string) error {
return m.PostFunc(url, contentType, body)
}
func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (string, error) {
return func(url string) (string, error) {
func toHttpHeader(header map[string]interface{}) http.Header {
httpHeader := http.Header{}
for key, value := range header {
httpHeader.Add(key, value.(string))
}
return httpHeader
}
func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (http.Header, string, error) {
return func(url string) (http.Header, string, error) {
fmt.Println(" - GET call at " + url)
// url: https://brainbaking.com/something-something.html
// want: ../../mocks/something-something.html
mockfile := relPath + strings.ReplaceAll(url, "https://brainbaking.com/", "")
@ -35,7 +52,15 @@ func RelPathGetBodyFunc(t *testing.T, relPath string) func(string) (string, erro
if err != nil {
t.Error(err)
}
return string(html), nil
headerData, headerFileErr := ioutil.ReadFile(strings.ReplaceAll(mockfile, ".html", "-headers.json"))
if headerFileErr != nil {
return http.Header{}, string(html), nil
}
headerJson := map[string]interface{}{}
json.Unmarshal(headerData, &headerJson)
return toHttpHeader(headerJson), string(html), nil
}
}

48
rest/client.go
View File

@ -4,46 +4,74 @@ import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
type Client interface {
Get(url string) (*http.Response, error)
Post(url string, contentType string, body string) error
GetBody(url string) (string, error)
GetBody(url string) (http.Header, string, error)
PostForm(url string, formData url.Values) error
}
type HttpClient struct {
}
func (client *HttpClient) Post(url string, contenType string, body string) error {
_, err := http.Post(url, contenType, strings.NewReader(body))
func (client *HttpClient) PostForm(url string, formData url.Values) error {
resp, err := http.PostForm(url, formData)
if err != nil {
return err
}
if !isStatusOk(resp) {
return fmt.Errorf("POST Form to %s: Status code is not OK (%d)", url, resp.StatusCode)
}
return nil
}
func (client *HttpClient) Post(url string, contenType string, body string) error {
resp, err := http.Post(url, contenType, strings.NewReader(body))
if err != nil {
return err
}
if !isStatusOk(resp) {
return fmt.Errorf("POST to %s: Status code is not OK (%d)", url, resp.StatusCode)
}
return nil
}
// something like this? https://freshman.tech/snippets/go/http-response-to-string/
func (client *HttpClient) GetBody(url string) (string, error) {
resp, geterr := http.Get(url)
func (client *HttpClient) GetBody(url string) (http.Header, string, error) {
resp, geterr := client.Get(url)
if geterr != nil {
return "", geterr
return nil, "", geterr
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return "", fmt.Errorf("Status code for %s is not OK (%d)", url, resp.StatusCode)
body, err := ReadBodyFromResponse(resp)
if err != nil {
return nil, "", err
}
return resp.Header, body, nil
}
func ReadBodyFromResponse(resp *http.Response) (string, error) {
if !isStatusOk(resp) {
return "", fmt.Errorf("Status code is not OK (%d)", resp.StatusCode)
}
defer resp.Body.Close()
body, readerr := ioutil.ReadAll(resp.Body)
defer resp.Body.Close()
if readerr != nil {
return "", readerr
}
return string(body), nil
}
func isStatusOk(resp *http.Response) bool {
return resp.StatusCode >= 200 && resp.StatusCode <= 299
}
func (client *HttpClient) Get(url string) (*http.Response, error) {
return http.Get(url)
}

8
rest/utils.go
View File

@ -2,6 +2,7 @@ package rest
import (
"net/http"
"net/url"
)
func BadRequest(w http.ResponseWriter) {
@ -12,3 +13,10 @@ func Accept(w http.ResponseWriter) {
w.WriteHeader(202)
w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!"))
}
// assumes the URL is well-formed.
func BaseUrlOf(link string) *url.URL {
obj, _ := url.Parse(link)
baseUrl, _ := url.Parse(obj.Scheme + "://" + obj.Host)
return baseUrl
}