From 942406a566c8157100d765380de7c250b4545dbb Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Tue, 27 Apr 2021 15:10:51 +0200 Subject: [PATCH] Revert "1.1.0 release: get rid of utcOffset" Why? Turns out that Pleroma logs dates in UTC... This reverts commit 6c9c5ff1e13769bea40dd470ea8aac1a7524d6f2. --- README.md | 11 ++-- RELEASE.md | 15 ------ package.json | 2 +- src/mastodon/feed-parser.js | 12 ++--- src/mastodon/templates.js | 3 +- .../masto-feed-invalid-publishedDate.xml | 50 ------------------- .../__snapshots__/feed-parser.test.js.snap | 6 +-- test/mastodon/feed-parser.test.js | 40 ++++++--------- 8 files changed, 28 insertions(+), 111 deletions(-) delete mode 100644 RELEASE.md delete mode 100644 test/__mocks__/masto-feed-invalid-publishedDate.xml diff --git a/README.md b/README.md index a0c3630..5fa251a 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,6 @@ Published at https://www.npmjs.com/package/jam-my-stack [![npm version](https://badge.fury.io/js/jam-my-stack.svg)](https://badge.fury.io/js/jam-my-stack) -Release notes: see `RELEASE.md` - ---- - These simple scripts **enrich your Jamstack-site** by adding/manipulating/whatever (meta)data, such as extra posts, indexing, and so forth. A primary example of these tools in action is my own site https://brainbaking.com - inspect how it's used at https://github.com/wgroeneveld/brainbaking **Are you looking for a way to receive webmentions?** See https://github.com/wgroeneveld/serve-my-jams ! @@ -33,6 +29,7 @@ Usage example: await mastodon.parseFeed({ notesdir: `${__dirname}/content/notes`, url: "https://chat.brainbaking.com/users/wouter/feed", + utcOffset: 60, titleCount: 50, titlePrefix: "Note: " }) @@ -40,6 +37,7 @@ Usage example: Options and their default values: +- `utcOffset`: `60` (= GMT+1, that's where I am!) (in **minutes**, see [day.js docs](https://day.js.org/docs/en/manipulate/utc-offset) - `titleCount`: `50`. Will add "..." and trim if title length bigger. - `titlePrefix`: `""`. Will add before title (e.g. "Note: ") - `ignoreReplies`: `false`. If true, will not process `in-reply-to` items. @@ -70,13 +68,13 @@ Example feed entry: ``` -This generates the file `01h19m03s35.md` with contents: +This generates the file `01h20m03s35.md` (it assumes UTC times in the feed and adjusts according to specified `utcOffset`, such as GMT+1 in this example), with contents: ```md --- source: "https://chat.brainbaking.com/objects/77a3ecfb-47e1-4d7a-a24a-8b779d80a8ac" title: "I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-ri..." -date: "2021-03-01T19:03:35+01:00" +date: "2021-03-01T19:03:35" --- I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ Very impressed so far! Also rely on my own CalDAV server to replace GCalendar. Any others here running #lineageos for privacy reasons? @@ -88,7 +86,6 @@ See implementation for more details and features. - `` image types (see `render-enclosures.ejs`) [ejs template](https://ejs.co/), that is appended to the Markdown file if any are found. Styling is up to you... - `... @https://blah.com/blie hi there` - this is a **in-reply-to** toot which adds `context` frontmatter, so your html renderer can use the correct IndieWeb classes. This should also enable webmention sending since you mention the URL. If you "at" a valid Mastodon user, it will automatically do this. -- On the `published` date: Mastodon feeds generate **ISO8601** timestamps with timezones. This is preserved in the resulting frontmatter using [dayjs](https://day.js.org/docs/en/display/format). ### 2. Goodreads diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index a94dc20..0000000 --- a/RELEASE.md +++ /dev/null @@ -1,15 +0,0 @@ - -# Jam-my-stack release notes - -### 1.1.0 - -- Finally get rid of `utcOffset` and include timezone in proper ISO8601 format in `date` front-matter. -- This means this release will probably cause filenames to change so watch out! - -### 1.0.17 - -- Implement @http... translation to `context` front-matter in Markdown output. - -### 1.0.16 - -- All the rest! diff --git a/package.json b/package.json index 20a218a..6023d8a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jam-my-stack", - "version": "1.1.0", + "version": "1.0.17", "repository": { "url": "https://github.com/wgroeneveld/jam-my-stack", "type": "git" diff --git a/src/mastodon/feed-parser.js b/src/mastodon/feed-parser.js index a0e1e6b..bad1471 100644 --- a/src/mastodon/feed-parser.js +++ b/src/mastodon/feed-parser.js @@ -26,8 +26,6 @@ function convertAtomItemToMd(item, notesdir) { if(!existsSync(`${notesdir}/${item.year}`)) mkdirSync(`${notesdir}/${item.year}`) if(!existsSync(path)) mkdirSync(path) - // dates are formatted using dayjs.format() in ISO8601 - // see https://day.js.org/docs/en/display/format let mddata = ejs.render(templates.markdown, { item }) if(item.media?.length > 0) { @@ -66,12 +64,13 @@ function detectContext(item, content) { // opts: // notesdir = `${__dirname}/content/notes` // url = "https://chat.brainbaking.com/users/wouter/feed"; +// utcOffset = 60 (in minutes) // titleCount = 50 // titlePrefix = "Note: " // ignoreReplies = false async function parseMastoFeed(options) { - const { notesdir, url, titleCount = 50, titlePrefix = "", ignoreReplies = false } = options + const { notesdir, url, utcOffset = 60, titleCount = 50, titlePrefix = "", ignoreReplies = false } = options const notesroot = await getFiles(notesdir) const notes = notesroot @@ -93,10 +92,7 @@ async function parseMastoFeed(options) { const items = entries.map(item => { const content = ent.decode(ent.decode(item.content['#text'])) // format: <span class="h-card.... - let date = dayjs(item.published) - if(!date.isValid()) { - date = dayjs.utc() - } + const date = dayjs.utc(item.published).utcOffset(utcOffset) const year = date.format("YYYY") const month = date.format("MM") const day = date.format("DD") @@ -118,7 +114,7 @@ async function parseMastoFeed(options) { id: stripBeforeLastSlash(item.id), media, hash: `${day}h${date.format("HH")}m${date.format("mm")}s${date.format("ss")}`, - date, // format: 2021-03-02T16:18:46.658056Z - KEEP timezones! + date, // format: 2021-03-02T16:18:46.658056Z year, month, day diff --git a/src/mastodon/templates.js b/src/mastodon/templates.js index b61ae58..1dcdd52 100644 --- a/src/mastodon/templates.js +++ b/src/mastodon/templates.js @@ -1,12 +1,11 @@ -// warning, without explicitly providing the ISO8601 format, dayjs.utc() formats differently!! const markdown = `--- source: "<%- item.url %>" <% if (item.context) { -%> context: "<%- item.context %>" <% } -%> title: "<%- item.title %>" -date: "<%- item.date.format('YYYY-MM-DDTHH:mm:ssZ') %>" +date: "<%- item.year %>-<%- item.month %>-<%- item.day %>T<%- item.date.format('HH:mm:ss') %>" --- <%- item.content %> diff --git a/test/__mocks__/masto-feed-invalid-publishedDate.xml b/test/__mocks__/masto-feed-invalid-publishedDate.xml deleted file mode 100644 index ca41a94..0000000 --- a/test/__mocks__/masto-feed-invalid-publishedDate.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - https://chat.brainbaking.com/users/wouter/feed.atom - wouter's timeline - 2021-03-02T16:18:46 - https://chat.brainbaking.com/media/f39bcd85-5098-45e2-b395-e274b712d512/headshot_2020.jpg - - - - https://chat.brainbaking.com/users/wouter - http://activitystrea.ms/schema/1.0/person - https://chat.brainbaking.com/users/wouter - wouter - Wouter Groeneveld - Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠. - Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠. - wouter - - - - - - true - - - - - - - - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - https://chat.brainbaking.com/objects/b5b67e88-eda8-45dd-ab8f-54443b62e250 - some title - <span class="h-card">@<a class="u-url mention" href="https://reply-to-stuff" rel="ugc"> in reply to previous url test - fuck - me - - tag:mastodon.social,2021-03-20:objectId=227433498:objectType=Conversation - - - - diff --git a/test/mastodon/__snapshots__/feed-parser.test.js.snap b/test/mastodon/__snapshots__/feed-parser.test.js.snap index 551061c..2d18d42 100644 --- a/test/mastodon/__snapshots__/feed-parser.test.js.snap +++ b/test/mastodon/__snapshots__/feed-parser.test.js.snap @@ -5,7 +5,7 @@ exports[`mastodon feed parser tests parse creates MD with context if in-reply-to source: \\"https://chat.brainbaking.com/objects/2e58289c-f5f0-415c-b2e1-62c74662aa16\\" context: \\"https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793\\" title: \\"@StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's ...\\" -date: \\"2021-03-02T17:18:46+01:00\\" +date: \\"2021-03-02T16:18:46\\" --- @StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's even cat hair inside the cases... (to be clear: also unintentional) @@ -16,7 +16,7 @@ exports[`mastodon feed parser tests parse creates correct MD structure 1`] = ` "--- source: \\"https://chat.brainbaking.com/objects/77a3ecfb-47e1-4d7a-a24a-8b779d80a8ac\\" title: \\"I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-ri...\\" -date: \\"2021-03-01T20:03:35+01:00\\" +date: \\"2021-03-01T19:03:35\\" --- I pulled the Google plug and installed LineageOS: https://brainbaking.com/post/2021/03/getting-rid-of-tracking-using-lineageos/ Very impressed so far! Also rely on my own CalDAV server to replace GCalendar. Any others here running #lineageos for privacy reasons? @@ -27,7 +27,7 @@ exports[`mastodon feed parser tests parse embedded images 1`] = ` "--- source: \\"https://chat.brainbaking.com/objects/a51e13ce-d618-4602-84f7-f398126510ff\\" title: \\"Enjoyed an afternoon of oldskool Diablo II on the ...\\" -date: \\"2021-03-14T17:41:53+01:00\\" +date: \\"2021-03-14T17:41:53\\" --- Enjoyed an afternoon of oldskool Diablo II on the Europebattle servers. We did a few Mephisto runs, managed to hit Hell, and I re-converetd my druid into a windy one. Good times! diff --git a/test/mastodon/feed-parser.test.js b/test/mastodon/feed-parser.test.js index 17410d5..5d5353d 100644 --- a/test/mastodon/feed-parser.test.js +++ b/test/mastodon/feed-parser.test.js @@ -1,7 +1,4 @@ -const MockDate = require('mockdate') -const dayjs = require('dayjs') - describe("mastodon feed parser tests", () => { const fs = require('fs'); const fsp = require('fs').promises; @@ -15,7 +12,6 @@ describe("mastodon feed parser tests", () => { const dumpdir = `${__dirname}/dump` beforeEach(() => { - MockDate.reset() if(fs.existsSync(dumpdir)) { rmdir(dumpdir) } @@ -55,23 +51,11 @@ describe("mastodon feed parser tests", () => { }) }) - test("uses now in UTC zone if published date is invalid", async () => { - MockDate.set(dayjs('2021-03-11T19:01:03+00:00').toDate()) - - await parseMastoFeed({ - url: "masto-feed-invalid-publishedDate", - notesdir: dumpdir - }) - - const actualMd = await fsp.readFile(`${dumpdir}/2021/03/11h19m01s03.md`) - const md = frontMatterParser.parseSync(actualMd.toString()) - expect(md.data.date).toBe('2021-03-11T19:01:03+00:00') - }) - test("parse embedded images", async () => { await parseMastoFeed({ url: "masto-feed-images", - notesdir: dumpdir + notesdir: dumpdir, + utcOffset: 1 }) const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/14h17m41s53.md`)).toString() @@ -81,10 +65,11 @@ describe("mastodon feed parser tests", () => { test("parse prepends double quotes with backlash to escape in frontmatter", async () => { await parseMastoFeed({ url: "masto-feed-quote", - notesdir: dumpdir + notesdir: dumpdir, + utcOffset: 0 }) - const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`) + const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`) const md = frontMatterParser.parseSync(actualMd.toString()) expect(md.data.title).toBe("\"wow this sucks\" with quotes") @@ -94,11 +79,12 @@ describe("mastodon feed parser tests", () => { await parseMastoFeed({ url: "masto-feed-sample", notesdir: dumpdir, + utcOffset: 0, titleCount: 5, titlePrefix: "Note: " }) - const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`) + const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`) const md = frontMatterParser.parseSync(actualMd.toString()) expect(md.data.title).toBe("Note: @Stam...") @@ -108,10 +94,11 @@ describe("mastodon feed parser tests", () => { await parseMastoFeed({ url: "masto-feed-sample", notesdir: dumpdir, + utcOffset: 0, titleCount: 5000 }) - const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`) + const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`) const md = frontMatterParser.parseSync(actualMd.toString()) expect(md.data.title).toBe("@StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There's ...") @@ -133,10 +120,11 @@ describe("mastodon feed parser tests", () => { await parseMastoFeed({ url: "masto-feed-sample", notesdir: dumpdir, + utcOffset: 0, titleCount: 5000 }) - const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h20m03s35.md`)).toString() + const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m03s35.md`)).toString() expect(actualMd).toMatchSnapshot() }) @@ -145,10 +133,11 @@ describe("mastodon feed parser tests", () => { await parseMastoFeed({ url: "masto-feed-sample", notesdir: dumpdir, + utcOffset: 0, titleCount: 5000 }) - const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/02h17m18s46.md`)).toString() + const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)).toString() expect(actualMd).toMatchSnapshot() const expectedReplyTo = "https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793" @@ -160,10 +149,11 @@ describe("mastodon feed parser tests", () => { await parseMastoFeed({ url: "masto-feed-at-url", notesdir: dumpdir, + utcOffset: 0, titleCount: 5000 }) - const actualMd = await fsp.readFile(`${dumpdir}/2021/03/20h12m12s08.md`) + const actualMd = await fsp.readFile(`${dumpdir}/2021/03/20h11m12s08.md`) const expectedReplyTo = "https://reply-to-stuff" const md = frontMatterParser.parseSync(actualMd.toString())