diff --git a/README.md b/README.md
index 5fa251a..a0c3630 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,10 @@ Published at https://www.npmjs.com/package/jam-my-stack
[![npm version](https://badge.fury.io/js/jam-my-stack.svg)](https://badge.fury.io/js/jam-my-stack)
+Release notes: see `RELEASE.md`
+
+---
+
These simple scripts **enrich your Jamstack-site** by adding/manipulating/whatever (meta)data, such as extra posts, indexing, and so forth. A primary example of these tools in action is my own site https://brainbaking.com - inspect how it's used at https://github.com/wgroeneveld/brainbaking
**Are you looking for a way to receive webmentions?** See https://github.com/wgroeneveld/serve-my-jams !
@@ -29,7 +33,6 @@ Usage example:
await mastodon.parseFeed({
notesdir: `${__dirname}/content/notes`,
url: "https://chat.brainbaking.com/users/wouter/feed",
- utcOffset: 60,
titleCount: 50,
titlePrefix: "Note: "
})
@@ -37,7 +40,6 @@ Usage example:
Options and their default values:
-- `utcOffset`: `60` (= GMT+1, that's where I am!) (in **minutes**, see [day.js docs](https://day.js.org/docs/en/manipulate/utc-offset)
- `titleCount`: `50`. Will add "..." and trim if title length bigger.
- `titlePrefix`: `""`. Will add before title (e.g. "Note: ")
- `ignoreReplies`: `false`. If true, will not process `in-reply-to` items.
@@ -68,13 +70,13 @@ Example feed entry:
```
-This generates the file `01h20m03s35.md` (it assumes UTC times in the feed and adjusts according to specified `utcOffset`, such as GMT+1 in this example), with contents:
+This generates the file `01h19m03s35.md` with contents:
```md
---
source: "https://chat.brainbaking.com/objects/77a3ecfb-47e1-4d7a-a24a-8b779d80a8ac"
title: "I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-ri..."
-date: "2021-03-01T19:03:35"
+date: "2021-03-01T19:03:35+01:00"
---
I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ Very impressed so far! Also rely on my own CalDAV server to replace GCalendar. Any others here running #lineageos for privacy reasons?
@@ -86,6 +88,7 @@ See implementation for more details and features.
- `` image types (see `render-enclosures.ejs`) [ejs template](https://ejs.co/), that is appended to the Markdown file if any are found. Styling is up to you...
- `... @https://blah.com/blie hi there` - this is a **in-reply-to** toot which adds `context` frontmatter, so your html renderer can use the correct IndieWeb classes. This should also enable webmention sending since you mention the URL. If you "at" a valid Mastodon user, it will automatically do this.
+- On the `published` date: Mastodon feeds generate **ISO8601** timestamps with timezones. This is preserved in the resulting frontmatter using [dayjs](https://day.js.org/docs/en/display/format).
### 2. Goodreads
diff --git a/RELEASE.md b/RELEASE.md
new file mode 100644
index 0000000..a94dc20
--- /dev/null
+++ b/RELEASE.md
@@ -0,0 +1,15 @@
+
+# Jam-my-stack release notes
+
+### 1.1.0
+
+- Finally get rid of `utcOffset` and include timezone in proper ISO8601 format in `date` front-matter.
+- This means this release will probably cause filenames to change so watch out!
+
+### 1.0.17
+
+- Implement @http... translation to `context` front-matter in Markdown output.
+
+### 1.0.16
+
+- All the rest!
diff --git a/package.json b/package.json
index 6023d8a..20a218a 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "jam-my-stack",
- "version": "1.0.17",
+ "version": "1.1.0",
"repository": {
"url": "https://github.com/wgroeneveld/jam-my-stack",
"type": "git"
diff --git a/src/mastodon/feed-parser.js b/src/mastodon/feed-parser.js
index bad1471..a0e1e6b 100644
--- a/src/mastodon/feed-parser.js
+++ b/src/mastodon/feed-parser.js
@@ -26,6 +26,8 @@ function convertAtomItemToMd(item, notesdir) {
if(!existsSync(`${notesdir}/${item.year}`)) mkdirSync(`${notesdir}/${item.year}`)
if(!existsSync(path)) mkdirSync(path)
+ // dates are formatted using dayjs.format() in ISO8601
+ // see https://day.js.org/docs/en/display/format
let mddata = ejs.render(templates.markdown, { item })
if(item.media?.length > 0) {
@@ -64,13 +66,12 @@ function detectContext(item, content) {
// opts:
// notesdir = `${__dirname}/content/notes`
// url = "https://chat.brainbaking.com/users/wouter/feed";
-// utcOffset = 60 (in minutes)
// titleCount = 50
// titlePrefix = "Note: "
// ignoreReplies = false
async function parseMastoFeed(options) {
- const { notesdir, url, utcOffset = 60, titleCount = 50, titlePrefix = "", ignoreReplies = false } = options
+ const { notesdir, url, titleCount = 50, titlePrefix = "", ignoreReplies = false } = options
const notesroot = await getFiles(notesdir)
const notes = notesroot
@@ -92,7 +93,10 @@ async function parseMastoFeed(options) {
const items = entries.map(item => {
const content = ent.decode(ent.decode(item.content['#text'])) // format: <span class="h-card....
- const date = dayjs.utc(item.published).utcOffset(utcOffset)
+ let date = dayjs(item.published)
+ if(!date.isValid()) {
+ date = dayjs.utc()
+ }
const year = date.format("YYYY")
const month = date.format("MM")
const day = date.format("DD")
@@ -114,7 +118,7 @@ async function parseMastoFeed(options) {
id: stripBeforeLastSlash(item.id),
media,
hash: `${day}h${date.format("HH")}m${date.format("mm")}s${date.format("ss")}`,
- date, // format: 2021-03-02T16:18:46.658056Z
+ date, // format: 2021-03-02T16:18:46.658056Z - KEEP timezones!
year,
month,
day
diff --git a/src/mastodon/templates.js b/src/mastodon/templates.js
index 1dcdd52..b61ae58 100644
--- a/src/mastodon/templates.js
+++ b/src/mastodon/templates.js
@@ -1,11 +1,12 @@
+// warning, without explicitly providing the ISO8601 format, dayjs.utc() formats differently!!
const markdown = `---
source: "<%- item.url %>"
<% if (item.context) { -%>
context: "<%- item.context %>"
<% } -%>
title: "<%- item.title %>"
-date: "<%- item.year %>-<%- item.month %>-<%- item.day %>T<%- item.date.format('HH:mm:ss') %>"
+date: "<%- item.date.format('YYYY-MM-DDTHH:mm:ssZ') %>"
---
<%- item.content %>
diff --git a/test/__mocks__/masto-feed-invalid-publishedDate.xml b/test/__mocks__/masto-feed-invalid-publishedDate.xml
new file mode 100644
index 0000000..ca41a94
--- /dev/null
+++ b/test/__mocks__/masto-feed-invalid-publishedDate.xml
@@ -0,0 +1,50 @@
+
+
+
+ https://chat.brainbaking.com/users/wouter/feed.atom
+ wouter's timeline
+ 2021-03-02T16:18:46
+ https://chat.brainbaking.com/media/f39bcd85-5098-45e2-b395-e274b712d512/headshot_2020.jpg
+
+
+
+ https://chat.brainbaking.com/users/wouter
+ http://activitystrea.ms/schema/1.0/person
+ https://chat.brainbaking.com/users/wouter
+ wouter
+ Wouter Groeneveld
+ Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠.
+ Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠.
+ wouter
+
+
+
+
+
+ true
+
+
+
+
+
+
+
+
+ http://activitystrea.ms/schema/1.0/note
+ http://activitystrea.ms/schema/1.0/post
+ https://chat.brainbaking.com/objects/b5b67e88-eda8-45dd-ab8f-54443b62e250
+ some title
+ <span class="h-card">@<a class="u-url mention" href="https://reply-to-stuff" rel="ugc"> in reply to previous url test
+ fuck
+ me
+
+ tag:mastodon.social,2021-03-20:objectId=227433498:objectType=Conversation
+
+
+
+
diff --git a/test/mastodon/__snapshots__/feed-parser.test.js.snap b/test/mastodon/__snapshots__/feed-parser.test.js.snap
index 2d18d42..551061c 100644
--- a/test/mastodon/__snapshots__/feed-parser.test.js.snap
+++ b/test/mastodon/__snapshots__/feed-parser.test.js.snap
@@ -5,7 +5,7 @@ exports[`mastodon feed parser tests parse creates MD with context if in-reply-to
source: \\"https://chat.brainbaking.com/objects/2e58289c-f5f0-415c-b2e1-62c74662aa16\\"
context: \\"https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793\\"
title: \\"@StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's ...\\"
-date: \\"2021-03-02T16:18:46\\"
+date: \\"2021-03-02T17:18:46+01:00\\"
---
@StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's even cat hair inside the cases... (to be clear: also unintentional)
@@ -16,7 +16,7 @@ exports[`mastodon feed parser tests parse creates correct MD structure 1`] = `
"---
source: \\"https://chat.brainbaking.com/objects/77a3ecfb-47e1-4d7a-a24a-8b779d80a8ac\\"
title: \\"I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-ri...\\"
-date: \\"2021-03-01T19:03:35\\"
+date: \\"2021-03-01T20:03:35+01:00\\"
---
I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ Very impressed so far! Also rely on my own CalDAV server to replace GCalendar. Any others here running #lineageos for privacy reasons?
@@ -27,7 +27,7 @@ exports[`mastodon feed parser tests parse embedded images 1`] = `
"---
source: \\"https://chat.brainbaking.com/objects/a51e13ce-d618-4602-84f7-f398126510ff\\"
title: \\"Enjoyed an afternoon of oldskool Diablo II on the ...\\"
-date: \\"2021-03-14T17:41:53\\"
+date: \\"2021-03-14T17:41:53+01:00\\"
---
Enjoyed an afternoon of oldskool Diablo II on the Europebattle servers. We did a few Mephisto runs, managed to hit Hell, and I re-converetd my druid into a windy one. Good times!
diff --git a/test/mastodon/feed-parser.test.js b/test/mastodon/feed-parser.test.js
index 5d5353d..17410d5 100644
--- a/test/mastodon/feed-parser.test.js
+++ b/test/mastodon/feed-parser.test.js
@@ -1,4 +1,7 @@
+const MockDate = require('mockdate')
+const dayjs = require('dayjs')
+
describe("mastodon feed parser tests", () => {
const fs = require('fs');
const fsp = require('fs').promises;
@@ -12,6 +15,7 @@ describe("mastodon feed parser tests", () => {
const dumpdir = `${__dirname}/dump`
beforeEach(() => {
+ MockDate.reset()
if(fs.existsSync(dumpdir)) {
rmdir(dumpdir)
}
@@ -51,11 +55,23 @@ describe("mastodon feed parser tests", () => {
})
})
+ test("uses now in UTC zone if published date is invalid", async () => {
+ MockDate.set(dayjs('2021-03-11T19:01:03+00:00').toDate())
+
+ await parseMastoFeed({
+ url: "masto-feed-invalid-publishedDate",
+ notesdir: dumpdir
+ })
+
+ const actualMd = await fsp.readFile(`${dumpdir}/2021/03/11h19m01s03.md`)
+ const md = frontMatterParser.parseSync(actualMd.toString())
+ expect(md.data.date).toBe('2021-03-11T19:01:03+00:00')
+ })
+
test("parse embedded images", async () => {
await parseMastoFeed({
url: "masto-feed-images",
- notesdir: dumpdir,
- utcOffset: 1
+ notesdir: dumpdir
})
const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/14h17m41s53.md`)).toString()
@@ -65,11 +81,10 @@ describe("mastodon feed parser tests", () => {
test("parse prepends double quotes with backlash to escape in frontmatter", async () => {
await parseMastoFeed({
url: "masto-feed-quote",
- notesdir: dumpdir,
- utcOffset: 0
+ notesdir: dumpdir
})
- const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)
+ const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`)
const md = frontMatterParser.parseSync(actualMd.toString())
expect(md.data.title).toBe("\"wow this sucks\" with quotes")
@@ -79,12 +94,11 @@ describe("mastodon feed parser tests", () => {
await parseMastoFeed({
url: "masto-feed-sample",
notesdir: dumpdir,
- utcOffset: 0,
titleCount: 5,
titlePrefix: "Note: "
})
- const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)
+ const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`)
const md = frontMatterParser.parseSync(actualMd.toString())
expect(md.data.title).toBe("Note: @Stam...")
@@ -94,11 +108,10 @@ describe("mastodon feed parser tests", () => {
await parseMastoFeed({
url: "masto-feed-sample",
notesdir: dumpdir,
- utcOffset: 0,
titleCount: 5000
})
- const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)
+ const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`)
const md = frontMatterParser.parseSync(actualMd.toString())
expect(md.data.title).toBe("@StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's ...")
@@ -120,11 +133,10 @@ describe("mastodon feed parser tests", () => {
await parseMastoFeed({
url: "masto-feed-sample",
notesdir: dumpdir,
- utcOffset: 0,
titleCount: 5000
})
- const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m03s35.md`)).toString()
+ const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h20m03s35.md`)).toString()
expect(actualMd).toMatchSnapshot()
})
@@ -133,11 +145,10 @@ describe("mastodon feed parser tests", () => {
await parseMastoFeed({
url: "masto-feed-sample",
notesdir: dumpdir,
- utcOffset: 0,
titleCount: 5000
})
- const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)).toString()
+ const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`)).toString()
expect(actualMd).toMatchSnapshot()
const expectedReplyTo = "https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793"
@@ -149,11 +160,10 @@ describe("mastodon feed parser tests", () => {
await parseMastoFeed({
url: "masto-feed-at-url",
notesdir: dumpdir,
- utcOffset: 0,
titleCount: 5000
})
- const actualMd = await fsp.readFile(`${dumpdir}/2021/03/20h11m12s08.md`)
+ const actualMd = await fsp.readFile(`${dumpdir}/2021/03/20h12m12s08.md`)
const expectedReplyTo = "https://reply-to-stuff"
const md = frontMatterParser.parseSync(actualMd.toString())