sanitize receiving publication dates, this caused yet another crash...
This commit is contained in:
parent
2812130d75
commit
40ae44b2fd
|
@ -121,6 +121,9 @@ Will result in a `202 Accepted` - it handles things async. Stores in `.json` fil
|
||||||
|
|
||||||
This also saves the author picture/avatar locally - if present in the microformat. It does _not_ resize images, however, if it's bigger than 5 MB, it falls back to a default one.
|
This also saves the author picture/avatar locally - if present in the microformat. It does _not_ resize images, however, if it's bigger than 5 MB, it falls back to a default one.
|
||||||
|
|
||||||
|
Publication dates are sanitized and stored in `published`. They should be formatted in ISO8601. See [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
|
||||||
|
If that is not the case, go-jamming falls back to the moment the mention was received.
|
||||||
|
|
||||||
#### 1.2 `GET /webmention/:domain/:token`
|
#### 1.2 `GET /webmention/:domain/:token`
|
||||||
|
|
||||||
Retrieves a JSON array with relevant webmentions stored for that domain. The token should match. See configuration to fiddle with it yourself.
|
Retrieves a JSON array with relevant webmentions stored for that domain. The token should match. See configuration to fiddle with it yourself.
|
||||||
|
|
|
@ -9,8 +9,24 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DateFormat = "2006-01-02T15:04:05"
|
dateFormatWithTimeZone = "2006-01-02T15:04:05-07:00"
|
||||||
Anonymous = "anonymous"
|
dateFormatWithAbsoluteTimeZone = "2006-01-02T15:04:05-0700"
|
||||||
|
dateFormatWithTimeZoneSuffixed = "2006-01-02T15:04:05.000Z"
|
||||||
|
dateFormatWithoutTimeZone = "2006-01-02T15:04:05"
|
||||||
|
dateFormatWithSecondsWithoutTimeZone = "2006-01-02T15:04:05.00Z"
|
||||||
|
dateFormatWithoutTime = "2006-01-02"
|
||||||
|
Anonymous = "anonymous"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
supportedFormats = []string{
|
||||||
|
dateFormatWithTimeZone,
|
||||||
|
dateFormatWithAbsoluteTimeZone,
|
||||||
|
dateFormatWithTimeZoneSuffixed,
|
||||||
|
dateFormatWithSecondsWithoutTimeZone,
|
||||||
|
dateFormatWithoutTimeZone,
|
||||||
|
dateFormatWithoutTime,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
type IndiewebAuthor struct {
|
type IndiewebAuthor struct {
|
||||||
|
@ -63,8 +79,8 @@ func (id *IndiewebData) IsEmpty() bool {
|
||||||
return id.Url == ""
|
return id.Url == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func PublishedNow(utcOffset int) string {
|
func PublishedNow(zone *time.Location) string {
|
||||||
return common.Now().UTC().Add(time.Duration(utcOffset) * time.Minute).Format("2006-01-02T15:04:05")
|
return common.Now().UTC().In(zone).Format(dateFormatWithTimeZone)
|
||||||
}
|
}
|
||||||
|
|
||||||
func shorten(txt string) string {
|
func shorten(txt string) string {
|
||||||
|
@ -145,12 +161,21 @@ func Prop(mf *microformats.Microformat, key string) *microformats.Microformat {
|
||||||
return mfEmpty()
|
return mfEmpty()
|
||||||
}
|
}
|
||||||
|
|
||||||
func Published(hEntry *microformats.Microformat, utcOffset int) string {
|
func Published(hEntry *microformats.Microformat, zone *time.Location) string {
|
||||||
publishedDate := Str(hEntry, "published")
|
publishedDate := Str(hEntry, "published")
|
||||||
if publishedDate == "" {
|
if publishedDate == "" {
|
||||||
return PublishedNow(utcOffset)
|
return PublishedNow(zone)
|
||||||
}
|
}
|
||||||
return publishedDate
|
|
||||||
|
for _, format := range supportedFormats {
|
||||||
|
formatted, err := time.Parse(format, publishedDate)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return formatted.Format(dateFormatWithTimeZone)
|
||||||
|
}
|
||||||
|
|
||||||
|
return PublishedNow(zone)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAuthor(hEntry *microformats.Microformat, hCard *microformats.Microformat) IndiewebAuthor {
|
func NewAuthor(hEntry *microformats.Microformat, hCard *microformats.Microformat) IndiewebAuthor {
|
||||||
|
|
|
@ -0,0 +1,100 @@
|
||||||
|
package mf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"brainbaking.com/go-jamming/common"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
"willnorris.com/go/microformats"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPublished(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
label string
|
||||||
|
raw string
|
||||||
|
expectedTime string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"Converts published date in RFC3339 ISO8601 indieweb datetime format with timezone",
|
||||||
|
"2021-04-25T11:24:48+02:00",
|
||||||
|
"2021-04-25T11:24:48+02:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Converts published date in RFC3339 ISO8601 indieweb datetime format with absolute timezone",
|
||||||
|
"2021-04-25T11:24:48+0200",
|
||||||
|
"2021-04-25T11:24:48+02:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Converts published date in RFC3339 ISO8601 indieweb datetime format with timezone suffixed with Z",
|
||||||
|
"2021-03-02T16:17:18.000Z",
|
||||||
|
"2021-03-02T16:17:18+00:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Converts published date in RFC3339 ISO8601 indieweb datetime format without timezone",
|
||||||
|
"2021-04-25T11:24:48",
|
||||||
|
"2021-04-25T11:24:48+00:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Converts published date in RFC3339 ISO8601 indieweb datetime format without time",
|
||||||
|
"2021-04-25",
|
||||||
|
"2021-04-25T00:00:00+00:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Returns current date if property with correct timezone not found",
|
||||||
|
"",
|
||||||
|
"2020-01-01T13:30:00+01:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Reverts to current date if not in correct ISO8601 datetime format",
|
||||||
|
"26 April 2021",
|
||||||
|
"2020-01-01T13:30:00+01:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"https://www.ietf.org/rfc/rfc3339.txt example 1",
|
||||||
|
"1985-04-12T23:20:50.52Z",
|
||||||
|
"1985-04-12T23:20:50+00:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"https://www.ietf.org/rfc/rfc3339.txt example 2",
|
||||||
|
"1996-12-19T16:39:57-08:00",
|
||||||
|
"1996-12-19T16:39:57-08:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"https://www.ietf.org/rfc/rfc3339.txt example 3 explicitly not implemented",
|
||||||
|
"1990-12-31T23:59:60Z",
|
||||||
|
"2020-01-01T13:30:00+01:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"https://www.ietf.org/rfc/rfc3339.txt example 4 explicitly not implemented",
|
||||||
|
"1990-12-31T15:59:60-08:00",
|
||||||
|
"2020-01-01T13:30:00+01:00",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"https://www.ietf.org/rfc/rfc3339.txt example 5 with seconds ignored",
|
||||||
|
"1937-01-01T12:00:27.87+00:20",
|
||||||
|
"1937-01-01T12:00:27+00:20",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
common.Now = func() time.Time {
|
||||||
|
return time.Date(2020, time.January, 1, 12, 30, 0, 0, time.UTC)
|
||||||
|
}
|
||||||
|
utcPlusOne := time.FixedZone("UTC+1", 60*60)
|
||||||
|
defer func() {
|
||||||
|
common.Now = time.Now
|
||||||
|
}()
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.label, func(t *testing.T) {
|
||||||
|
props := map[string][]interface{}{}
|
||||||
|
props["published"] = []interface{}{
|
||||||
|
tc.raw,
|
||||||
|
}
|
||||||
|
theTime := Published(µformats.Microformat{
|
||||||
|
Properties: props,
|
||||||
|
}, utcPlusOne)
|
||||||
|
|
||||||
|
assert.Equal(t, tc.expectedTime, theTime)
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -85,7 +85,7 @@ func (recv *Receiver) parseBodyAsIndiewebSite(hEntry *microformats.Microformat,
|
||||||
Author: mf.NewAuthor(hEntry, hCard),
|
Author: mf.NewAuthor(hEntry, hCard),
|
||||||
Content: mf.Content(hEntry),
|
Content: mf.Content(hEntry),
|
||||||
Url: mf.Url(hEntry, wm.Source),
|
Url: mf.Url(hEntry, wm.Source),
|
||||||
Published: mf.Published(hEntry, recv.Conf.UtcOffset),
|
Published: mf.Published(hEntry, recv.Conf.Zone()),
|
||||||
Source: wm.Source,
|
Source: wm.Source,
|
||||||
Target: wm.Target,
|
Target: wm.Target,
|
||||||
IndiewebType: mf.Type(hEntry),
|
IndiewebType: mf.Type(hEntry),
|
||||||
|
@ -100,7 +100,7 @@ func (recv *Receiver) parseBodyAsNonIndiewebSite(body string, wm mf.Mention) *mf
|
||||||
},
|
},
|
||||||
Name: title,
|
Name: title,
|
||||||
Content: title,
|
Content: title,
|
||||||
Published: mf.PublishedNow(recv.Conf.UtcOffset),
|
Published: mf.PublishedNow(recv.Conf.Zone()),
|
||||||
Url: wm.Source,
|
Url: wm.Source,
|
||||||
IndiewebType: mf.TypeMention,
|
IndiewebType: mf.TypeMention,
|
||||||
Source: wm.Source,
|
Source: wm.Source,
|
||||||
|
|
|
@ -126,7 +126,7 @@ func TestReceive(t *testing.T) {
|
||||||
Source: "https://brainbaking.com/valid-bridgy-twitter-source.html",
|
Source: "https://brainbaking.com/valid-bridgy-twitter-source.html",
|
||||||
Target: "https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag",
|
Target: "https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag",
|
||||||
},
|
},
|
||||||
json: `{"author":{"name":"Jamie Tanna","picture":"/pictures/brainbaking.com"},"name":"","content":"Recommended read:\nThe IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions\nhttps://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/","published":"2021-03-15T12:42:00+0000","url":"https://brainbaking.com/mf2/2021/03/1bkre/","type":"bookmark","source":"https://brainbaking.com/valid-bridgy-twitter-source.html","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag"}`,
|
json: `{"author":{"name":"Jamie Tanna","picture":"/pictures/brainbaking.com"},"name":"","content":"Recommended read:\nThe IndieWeb Mixed Bag - Thoughts about the (d)evolution of blog interactions\nhttps://brainbaking.com/post/2021/03/the-indieweb-mixed-bag/","published":"2021-03-15T12:42:00+00:00","url":"https://brainbaking.com/mf2/2021/03/1bkre/","type":"bookmark","source":"https://brainbaking.com/valid-bridgy-twitter-source.html","target":"https://brainbaking.com/post/2021/03/the-indieweb-mixed-bag"}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "receive a brid.gy Webmention like",
|
label: "receive a brid.gy Webmention like",
|
||||||
|
@ -136,7 +136,7 @@ func TestReceive(t *testing.T) {
|
||||||
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
||||||
},
|
},
|
||||||
// no dates in bridgy-to-mastodon likes...
|
// no dates in bridgy-to-mastodon likes...
|
||||||
json: `{"author":{"name":"Stampeding Longhorn","picture":"/pictures/brainbaking.com"},"name":"","content":"","published":"2020-01-01T12:30:00","url":"https://chat.brainbaking.com/notice/A4nx1rFwKUJYSe4TqK#favorited-by-A4nwg4LYyh4WgrJOXg","type":"like","source":"https://brainbaking.com/valid-bridgy-like.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`,
|
json: `{"author":{"name":"Stampeding Longhorn","picture":"/pictures/brainbaking.com"},"name":"","content":"","published":"2020-01-01T12:30:00+00:00","url":"https://chat.brainbaking.com/notice/A4nx1rFwKUJYSe4TqK#favorited-by-A4nwg4LYyh4WgrJOXg","type":"like","source":"https://brainbaking.com/valid-bridgy-like.html","target":"https://brainbaking.com/valid-indieweb-target.html"}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "receive a brid.gy Webmention that has a url and photo without value",
|
label: "receive a brid.gy Webmention that has a url and photo without value",
|
||||||
|
@ -144,7 +144,7 @@ func TestReceive(t *testing.T) {
|
||||||
Source: "https://brainbaking.com/valid-bridgy-source.html",
|
Source: "https://brainbaking.com/valid-bridgy-source.html",
|
||||||
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
||||||
},
|
},
|
||||||
json: `{"author":{"name":"Stampeding Longhorn", "picture":"/pictures/brainbaking.com"}, "content":"@wouter The cat pictures are awesome. for jest tests!", "name":"@wouter The cat pictures are awesome. for jest tests!", "published":"2021-03-02T16:17:18.000Z", "source":"https://brainbaking.com/valid-bridgy-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://social.linux.pizza/@StampedingLonghorn/105821099684887793"}`,
|
json: `{"author":{"name":"Stampeding Longhorn", "picture":"/pictures/brainbaking.com"}, "content":"@wouter The cat pictures are awesome. for jest tests!", "name":"@wouter The cat pictures are awesome. for jest tests!", "published":"2021-03-02T16:17:18+00:00", "source":"https://brainbaking.com/valid-bridgy-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://social.linux.pizza/@StampedingLonghorn/105821099684887793"}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "receive saves a JSON file of indieweb-metadata if all is valid",
|
label: "receive saves a JSON file of indieweb-metadata if all is valid",
|
||||||
|
@ -152,7 +152,7 @@ func TestReceive(t *testing.T) {
|
||||||
Source: "https://brainbaking.com/valid-indieweb-source.html",
|
Source: "https://brainbaking.com/valid-indieweb-source.html",
|
||||||
Target: "https://jefklakscodex.com/articles",
|
Target: "https://jefklakscodex.com/articles",
|
||||||
},
|
},
|
||||||
json: `{"author":{"name":"Wouter Groeneveld","picture":"/pictures/brainbaking.com"},"name":"I just learned about https://www.inklestudios.com/...","content":"This is cool, I just found out about valid indieweb target - so cool","published":"2021-03-06T12:41:00","url":"https://brainbaking.com/notes/2021/03/06h12m41s48/","type":"mention","source":"https://brainbaking.com/valid-indieweb-source.html","target":"https://jefklakscodex.com/articles"}`,
|
json: `{"author":{"name":"Wouter Groeneveld","picture":"/pictures/brainbaking.com"},"name":"I just learned about https://www.inklestudios.com/...","content":"This is cool, I just found out about valid indieweb target - so cool","published":"2021-03-06T12:41:00+00:00","url":"https://brainbaking.com/notes/2021/03/06h12m41s48/","type":"mention","source":"https://brainbaking.com/valid-indieweb-source.html","target":"https://jefklakscodex.com/articles"}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "receive saves a JSON file of indieweb-metadata with summary as content if present",
|
label: "receive saves a JSON file of indieweb-metadata with summary as content if present",
|
||||||
|
@ -160,7 +160,7 @@ func TestReceive(t *testing.T) {
|
||||||
Source: "https://brainbaking.com/valid-indieweb-source-with-summary.html",
|
Source: "https://brainbaking.com/valid-indieweb-source-with-summary.html",
|
||||||
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
||||||
},
|
},
|
||||||
json: `{"author":{"name":"Wouter Groeneveld", "picture":"/pictures/brainbaking.com"}, "content":"This is cool, this is a summary!", "name":"I just learned about https://www.inklestudios.com/...", "published":"2021-03-06T12:41:00", "source":"https://brainbaking.com/valid-indieweb-source-with-summary.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/notes/2021/03/06h12m41s48/"}`,
|
json: `{"author":{"name":"Wouter Groeneveld", "picture":"/pictures/brainbaking.com"}, "content":"This is cool, this is a summary!", "name":"I just learned about https://www.inklestudios.com/...", "published":"2021-03-06T12:41:00+00:00", "source":"https://brainbaking.com/valid-indieweb-source-with-summary.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/notes/2021/03/06h12m41s48/"}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "receive saves a JSON file of non-indieweb-data such as title if all is valid",
|
label: "receive saves a JSON file of non-indieweb-data such as title if all is valid",
|
||||||
|
@ -168,7 +168,7 @@ func TestReceive(t *testing.T) {
|
||||||
Source: "https://brainbaking.com/valid-nonindieweb-source.html",
|
Source: "https://brainbaking.com/valid-nonindieweb-source.html",
|
||||||
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
Target: "https://brainbaking.com/valid-indieweb-target.html",
|
||||||
},
|
},
|
||||||
json: `{"author":{"name":"https://brainbaking.com/valid-nonindieweb-source.html", "picture":""}, "content":"Diablo 2 Twenty Years Later: A Retrospective | Jefklaks Codex", "name":"Diablo 2 Twenty Years Later: A Retrospective | Jefklaks Codex", "published":"2020-01-01T12:30:00", "source":"https://brainbaking.com/valid-nonindieweb-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/valid-nonindieweb-source.html"}`,
|
json: `{"author":{"name":"https://brainbaking.com/valid-nonindieweb-source.html", "picture":""}, "content":"Diablo 2 Twenty Years Later: A Retrospective | Jefklaks Codex", "name":"Diablo 2 Twenty Years Later: A Retrospective | Jefklaks Codex", "published":"2020-01-01T12:30:00+00:00", "source":"https://brainbaking.com/valid-nonindieweb-source.html", "target":"https://brainbaking.com/valid-indieweb-target.html", "type":"mention", "url":"https://brainbaking.com/valid-nonindieweb-source.html"}`,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
|
@ -18,6 +19,10 @@ type Config struct {
|
||||||
DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
|
DisallowedWebmentionDomains []string `json:"disallowedWebmentionDomains"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) Zone() *time.Location {
|
||||||
|
return time.FixedZone("local", c.UtcOffset*60)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Config) missingKeys() []string {
|
func (c *Config) missingKeys() []string {
|
||||||
keys := []string{}
|
keys := []string{}
|
||||||
if c.Port == 0 {
|
if c.Port == 0 {
|
||||||
|
|
336
mentions.db
336
mentions.db
|
@ -376,3 +376,339 @@ $21
|
||||||
brainbaking.com:since
|
brainbaking.com:since
|
||||||
$24
|
$24
|
||||||
2021-04-19T15:51:50.183Z
|
2021-04-19T15:51:50.183Z
|
||||||
|
*3
|
||||||
|
$3
|
||||||
|
set
|
||||||
|
$21
|
||||||
|
brainbaking.com:since
|
||||||
|
$24
|
||||||
|
2021-04-19T17:02:07.277Z
|
||||||
|
*3
|
||||||
|
$3
|
||||||
|
set
|
||||||
|
$21
|
||||||
|
brainbaking.com:since
|
||||||
|
$24
|
||||||
|
2021-04-19T17:55:47.794Z
|
||||||
|
*3
|
||||||
|
$3
|
||||||
|
set
|
||||||
|
$23
|
||||||
|
brainbaking.com:picture
|
||||||
|
$4861
|
||||||
|
ÿØÿà |