diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..73f69e0
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/
diff --git a/.idea/go-jamming.iml b/.idea/go-jamming.iml
new file mode 100644
index 0000000..c956989
--- /dev/null
+++ b/.idea/go-jamming.iml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..cb3f101
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/app/webmention/microformats.go b/app/webmention/microformats.go
new file mode 100644
index 0000000..e69827c
--- /dev/null
+++ b/app/webmention/microformats.go
@@ -0,0 +1,47 @@
+package webmention
+
+import "willnorris.com/go/microformats"
+
+// Go stuff: entry.Properties["name"][0].(string),
+// JS stuff: hEntry.properties?.name?.[0]
+// The problem: convoluted syntax and no optional chaining!
+func mfstr(mf *microformats.Microformat, key string) string {
+ val := mf.Properties[key]
+ if len(val) == 0 {
+ return ""
+ }
+
+ str, ok := val[0].(string)
+ if !ok {
+ // in very weird cases, it could be a map holding a value, like in mf2's "photo"
+ valMap, ok2 := val[0].(map[string]string)
+ if !ok2 {
+ str = ""
+ }
+ str = valMap["value"]
+ }
+
+ return str
+}
+
+func mfmap(mf *microformats.Microformat, key string) map[string]string {
+ val := mf.Properties[key]
+ if len(val) == 0 {
+ return map[string]string{}
+ }
+ mapVal, ok := val[0].(map[string]string)
+ if !ok {
+ return map[string]string{}
+ }
+ return mapVal
+}
+
+func mfprop(mf *microformats.Microformat, key string) *microformats.Microformat {
+ val := mf.Properties[key]
+ if len(val) == 0 {
+ return µformats.Microformat{
+ Properties: map[string][]interface{}{},
+ }
+ }
+ return val[0].(*microformats.Microformat)
+}
diff --git a/app/webmention/receive.go b/app/webmention/receive.go
index 1fbbcb7..e0d5b6a 100644
--- a/app/webmention/receive.go
+++ b/app/webmention/receive.go
@@ -3,6 +3,7 @@ package webmention
import (
"fmt"
+ "net/url"
"strings"
"os"
"crypto/md5"
@@ -29,6 +30,11 @@ func (wm *webmention) asPath(conf *common.Config) string {
return conf.DataPath + "/" + domain + "/" + filename + ".json"
}
+func (wm *webmention) sourceUrl() *url.URL {
+ url, _ := url.Parse(wm.source)
+ return url
+}
+
// used as a "class" to iject dependencies, just to be able to test. Do NOT like htis.
// Is there a better way? e.g. in validate, I just pass rest.Client as an arg. Not great either.
type receiver struct {
@@ -53,16 +59,99 @@ func (recv *receiver) deletePossibleOlderWebmention(wm webmention) {
os.Remove(wm.asPath(recv.conf))
}
+func getHEntry(data *microformats.Data) *microformats.Microformat {
+ for _, itm := range data.Items {
+ if common.Includes(itm.Type, "h-entry") {
+ return itm
+ }
+ }
+ return nil
+}
+
+type indiewebAuthor struct {
+ name string
+ picture string
+}
+
+type indiewebData struct {
+ author indiewebAuthor
+ name string
+ content string
+ published string // TODO to a date
+ url string
+ dateType string // TODO json property "type"
+ source string
+ target string
+}
+
func (recv *receiver) processSourceBody(body string, wm webmention) {
- if strings.Index(body, wm.target) == -1 {
+ if !strings.Contains(body, wm.target) {
log.Warn().Str("target", wm.target).Msg("ABORT: no mention of target found in html src of source!")
return
}
r := strings.NewReader(body)
- data := microformats.Parse(r, nil)
-
- fmt.Println(data.Items[0].Type[0]) // h-entry
- // then: .Properties on Items[0]
- // see https://github.com/willnorris/microformats/blob/main/microformats.go
+ data := microformats.Parse(r, wm.sourceUrl())
+ hEntry := getHEntry(data)
+ var indieweb *indiewebData
+ if hEntry == nil {
+ indieweb = parseBodyAsNonIndiewebSite(body, wm)
+ } else {
+ indieweb = parseBodyAsIndiewebSite(hEntry, wm)
+ }
+
+ saveWebmentionToDisk(wm, indieweb)
+ log.Info().Str("file", wm.asPath(recv.conf)).Msg("OK: webmention processed.")
+}
+
+func saveWebmentionToDisk(wm webmention, indieweb *indiewebData) {
+
+}
+
+// TODO I'm smelling very unstable code, apply https://golang.org/doc/effective_go#recover here?
+// see https://github.com/willnorris/microformats/blob/main/microformats.go
+func parseBodyAsIndiewebSite(hEntry *microformats.Microformat, wm webmention) *indiewebData {
+ name := mfstr(hEntry, "name")
+ authorName := mfstr(mfprop(hEntry, "author"), "name")
+ if authorName == "" {
+ authorName = mfprop(hEntry, "author").Value
+ }
+ // TODO sometimes it's picture.value??
+ pic := mfstr(mfprop(hEntry, "author"), "photo")
+ summary := mfstr(hEntry, "summary")
+ contentEntry := mfmap(hEntry, "content")["value"]
+ bridgyTwitterContent := mfstr(hEntry, "bridgy-twitter-content")
+
+ return &indiewebData{
+ name: name,
+ author: indiewebAuthor{
+ name: authorName,
+ picture: pic,
+ },
+ content: determineContent(summary, contentEntry, bridgyTwitterContent),
+ source: wm.source,
+ target: wm.target,
+ }
+
+ //len(entry.Properties["hoopw"])
+}
+
+func shorten(txt string) string {
+ if len(txt) <= 250 {
+ return txt
+ }
+ return txt[0:250] + "..."
+}
+
+func determineContent(summary string, contentEntry string, bridgyTwitterContent string) string {
+ if bridgyTwitterContent != "" {
+ return shorten(bridgyTwitterContent)
+ } else if summary != "" {
+ return shorten(summary)
+ }
+ return shorten(contentEntry)
+}
+
+func parseBodyAsNonIndiewebSite(body string, wm webmention) *indiewebData {
+ return nil
}
diff --git a/app/webmention/receive_test.go b/app/webmention/receive_test.go
index 867ea00..fb27b40 100644
--- a/app/webmention/receive_test.go
+++ b/app/webmention/receive_test.go
@@ -2,9 +2,9 @@
package webmention
import (
- "testing"
- "os"
"errors"
+ "os"
+ "testing"
"github.com/wgroeneveld/go-jamming/common"
"github.com/wgroeneveld/go-jamming/mocks"
@@ -36,6 +36,26 @@ func writeSomethingTo(filename string) {
defer file.Close()
}
+func TestReceiveTargetExistsSavesWebmentionToDisk(t *testing.T) {
+ os.MkdirAll("testdata/jefklakscodex.com", os.ModePerm)
+ //defer os.RemoveAll("testdata")
+
+ wm := webmention{
+ source: "https://brainbaking.com",
+ target: "https://jefklakscodex.com/articles",
+ }
+ //filename := wm.asPath(conf)
+
+ receiver := &receiver {
+ conf: conf,
+ restClient: &mocks.RestClientMock{
+ GetBodyFunc: mocks.BodyFunc(t, "../../mocks/valid-indieweb-source.html"),
+ },
+ }
+
+ receiver.receive(wm)
+}
+
func TestReceiveTargetDoesNotExistAnymoreDeletesPossiblyOlderWebmention(t *testing.T) {
os.MkdirAll("testdata/jefklakscodex.com", os.ModePerm)
defer os.RemoveAll("testdata")
diff --git a/common/config.go b/common/config.go
index d2caf1f..a3c82d7 100644
--- a/common/config.go
+++ b/common/config.go
@@ -21,7 +21,7 @@ type Config struct {
func (c *Config) FetchDomain(url string) (string, error) {
for _, domain := range c.AllowedWebmentionSources {
- if strings.Index(url, domain) != -1 {
+ if strings.Contains(url, domain) {
return domain, nil
}
}
diff --git a/common/slices.go b/common/slices.go
new file mode 100644
index 0000000..bde04ff
--- /dev/null
+++ b/common/slices.go
@@ -0,0 +1,10 @@
+package common
+
+func Includes(slice []string, elem string) bool {
+ for _, el := range slice {
+ if el == elem {
+ return true
+ }
+ }
+ return false
+}
diff --git a/common/slices_test.go b/common/slices_test.go
new file mode 100644
index 0000000..07c2a31
--- /dev/null
+++ b/common/slices_test.go
@@ -0,0 +1,21 @@
+package common
+
+import "testing"
+
+func TestIncludesElemInArrayTrue(t *testing.T) {
+ arr := []string{ "one", "two"}
+
+ result := Includes(arr, "two")
+ if result != true {
+ t.Error("Should be in there")
+ }
+}
+
+func TestIncludesElemNotInArrayFalse(t *testing.T) {
+ arr := []string{ "one", "two"}
+
+ result := Includes(arr, "three")
+ if result != false {
+ t.Error("Should NOT be in there")
+ }
+}
diff --git a/jsfork/test/__mocks__/index.xml b/mocks/index.xml
similarity index 100%
rename from jsfork/test/__mocks__/index.xml
rename to mocks/index.xml
diff --git a/jsfork/test/__mocks__/link-discover-bothtypes.html b/mocks/link-discover-bothtypes.html
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-bothtypes.html
rename to mocks/link-discover-bothtypes.html
diff --git a/jsfork/test/__mocks__/link-discover-test-headers.json b/mocks/link-discover-test-headers.json
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-test-headers.json
rename to mocks/link-discover-test-headers.json
diff --git a/jsfork/test/__mocks__/link-discover-test-multiple.html b/mocks/link-discover-test-multiple.html
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-test-multiple.html
rename to mocks/link-discover-test-multiple.html
diff --git a/jsfork/test/__mocks__/link-discover-test-none.html b/mocks/link-discover-test-none.html
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-test-none.html
rename to mocks/link-discover-test-none.html
diff --git a/jsfork/test/__mocks__/link-discover-test-single.html b/mocks/link-discover-test-single.html
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-test-single.html
rename to mocks/link-discover-test-single.html
diff --git a/jsfork/test/__mocks__/link-discover-test.html b/mocks/link-discover-test.html
similarity index 100%
rename from jsfork/test/__mocks__/link-discover-test.html
rename to mocks/link-discover-test.html
diff --git a/jsfork/test/__mocks__/pingback-discover-test-headers.json b/mocks/pingback-discover-test-headers.json
similarity index 100%
rename from jsfork/test/__mocks__/pingback-discover-test-headers.json
rename to mocks/pingback-discover-test-headers.json
diff --git a/jsfork/test/__mocks__/pingback-discover-test-multiple.html b/mocks/pingback-discover-test-multiple.html
similarity index 100%
rename from jsfork/test/__mocks__/pingback-discover-test-multiple.html
rename to mocks/pingback-discover-test-multiple.html
diff --git a/jsfork/test/__mocks__/pingback-discover-test-single.html b/mocks/pingback-discover-test-single.html
similarity index 100%
rename from jsfork/test/__mocks__/pingback-discover-test-single.html
rename to mocks/pingback-discover-test-single.html
diff --git a/jsfork/test/__mocks__/pingback-discover-test.html b/mocks/pingback-discover-test.html
similarity index 100%
rename from jsfork/test/__mocks__/pingback-discover-test.html
rename to mocks/pingback-discover-test.html
diff --git a/mocks/restclient.go b/mocks/restclient.go
index 6663aa8..31f5329 100644
--- a/mocks/restclient.go
+++ b/mocks/restclient.go
@@ -2,6 +2,8 @@
package mocks
import (
+ "testing"
+ "io/ioutil"
"net/http"
)
@@ -19,3 +21,12 @@ func (m *RestClientMock) GetBody(url string) (string, error) {
return m.GetBodyFunc(url)
}
+func BodyFunc(t *testing.T, mockfile string) func(string) (string, error) {
+ html, err := ioutil.ReadFile(mockfile)
+ if err != nil {
+ t.Error(err)
+ }
+ return func(url string) (string, error) {
+ return string(html), nil
+ }
+}
diff --git a/jsfork/test/__mocks__/samplerss-updated-timestamp.xml b/mocks/samplerss-updated-timestamp.xml
similarity index 100%
rename from jsfork/test/__mocks__/samplerss-updated-timestamp.xml
rename to mocks/samplerss-updated-timestamp.xml
diff --git a/jsfork/test/__mocks__/samplerss.xml b/mocks/samplerss.xml
similarity index 100%
rename from jsfork/test/__mocks__/samplerss.xml
rename to mocks/samplerss.xml
diff --git a/jsfork/test/__mocks__/valid-bridgy-like.html b/mocks/valid-bridgy-like.html
similarity index 100%
rename from jsfork/test/__mocks__/valid-bridgy-like.html
rename to mocks/valid-bridgy-like.html
diff --git a/jsfork/test/__mocks__/valid-bridgy-source.html b/mocks/valid-bridgy-source.html
similarity index 100%
rename from jsfork/test/__mocks__/valid-bridgy-source.html
rename to mocks/valid-bridgy-source.html
diff --git a/jsfork/test/__mocks__/valid-bridgy-twitter-source.html b/mocks/valid-bridgy-twitter-source.html
similarity index 100%
rename from jsfork/test/__mocks__/valid-bridgy-twitter-source.html
rename to mocks/valid-bridgy-twitter-source.html
diff --git a/jsfork/test/__mocks__/valid-indieweb-source-with-summary.html b/mocks/valid-indieweb-source-with-summary.html
similarity index 100%
rename from jsfork/test/__mocks__/valid-indieweb-source-with-summary.html
rename to mocks/valid-indieweb-source-with-summary.html
diff --git a/jsfork/test/__mocks__/valid-indieweb-source.html b/mocks/valid-indieweb-source.html
similarity index 99%
rename from jsfork/test/__mocks__/valid-indieweb-source.html
rename to mocks/valid-indieweb-source.html
index 8dde647..0c64f54 100644
--- a/jsfork/test/__mocks__/valid-indieweb-source.html
+++ b/mocks/valid-indieweb-source.html
@@ -133,6 +133,7 @@
+ Check this out: cool stuff!