From 896b9658797403295c9735d7e45d3b01fdb2ee1f Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Fri, 5 Mar 2021 16:55:32 +0100 Subject: [PATCH] use moment.js insted of parsing dates yourself... --- .pnp.js | 11 +++++++++++ README.md | 9 +++++++-- jest.config.js | 4 +++- package.json | 3 ++- src/masto-feed-parser.js | 35 +++++++++++++++++----------------- test/masto-feed-parser.test.js | 8 +++++--- test/moment.test.js | 13 +++++++++++++ yarn.lock | 8 ++++++++ 8 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 test/moment.test.js diff --git a/.pnp.js b/.pnp.js index c21016a..e4de20d 100755 --- a/.pnp.js +++ b/.pnp.js @@ -43,6 +43,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["howlongtobeat", "npm:1.3.1"], ["jest", "npm:26.6.3"], ["lunr", "npm:2.3.9"], + ["moment", "npm:2.29.1"], ["parser-front-matter", "npm:1.6.4"] ], "linkType": "SOFT", @@ -3523,6 +3524,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["howlongtobeat", "npm:1.3.1"], ["jest", "npm:26.6.3"], ["lunr", "npm:2.3.9"], + ["moment", "npm:2.29.1"], ["parser-front-matter", "npm:1.6.4"] ], "linkType": "SOFT", @@ -4505,6 +4507,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD", }] ]], + ["moment", [ + ["npm:2.29.1", { + "packageLocation": "./.yarn/cache/moment-npm-2.29.1-787d9fdafd-86729013fe.zip/node_modules/moment/", + "packageDependencies": [ + ["moment", "npm:2.29.1"] + ], + "linkType": "HARD", + }] + ]], ["ms", [ ["npm:2.0.0", { "packageLocation": "./.yarn/cache/ms-npm-2.0.0-9e1101a471-1a230340cc.zip/node_modules/ms/", diff --git a/README.md b/README.md index 2fedf24..6a7fd9c 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,15 @@ Usage example: ```js await mastodon.parseFeed({ notesdir: `${__dirname}/content/notes`, - url: "https://chat.brainbaking.com/users/wouter/feed" + url: "https://chat.brainbaking.com/users/wouter/feed", + utcOffset: "+01:00" }) ``` +Default values: + +- `utcOffset`: `"+01:00"` (because that's where I am!) + Note that this **does not** delete the notes dir with every call. It simply checks if there isn't already a file with the same name (based on the publication date), and adds one if not. Example feed entry: @@ -56,7 +61,7 @@ Example feed entry: ``` -This generates the file `01h20m3s35.md` (it uses your local timezone based on the `` tag - mine is GMT+1) with contents: +This generates the file `01h20m03s35.md` (it assumes UTC times in the feed and adjusts according to specified `utcOffset`, such as GMT+1 in this example), with contents: ```md --- diff --git a/jest.config.js b/jest.config.js index 432d473..8480333 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1 +1,3 @@ -process.env.TZ = 'GMT'; + +// doing this on purpose to force test and implementation to use corret moment.js parsing +process.env.TZ = 'GMT+6'; diff --git a/package.json b/package.json index 9564830..03fee9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jam-my-stack", - "version": "1.0.3", + "version": "1.0.4", "repository": { "url": "https://github.com/wgroeneveld/jam-my-stack", "type": "git" @@ -17,6 +17,7 @@ "got": "^11.8.2", "howlongtobeat": "^1.3.1", "lunr": "^2.3.9", + "moment": "^2.29.1", "parser-front-matter": "^1.6.4" }, "devDependencies": { diff --git a/src/masto-feed-parser.js b/src/masto-feed-parser.js index 454d6ca..75d8d08 100644 --- a/src/masto-feed-parser.js +++ b/src/masto-feed-parser.js @@ -3,7 +3,7 @@ const parser = require("fast-xml-parser"); const { writeFileSync, existsSync, mkdirSync } = require('fs'); const ent = require('ent') const { getFiles } = require('./file-utils'); - +const moment = require('moment') function stripBeforeThirdSlash(str) { const splitted = str.split('/') @@ -14,20 +14,16 @@ function stripBeforeLastSlash(str) { return str.substring(str.lastIndexOf('/') + 1, str.length) } -function strpad(n) { - return String("0" + n).slice(-2); -} - -function convertAtomItemToMd(item, opts) { - const path = `${opts.notesdir}/${item.year}/${item.month}` - if(!existsSync(`${opts.notesdir}/${item.year}`)) mkdirSync(`${opts.notesdir}/${item.year}`) +function convertAtomItemToMd(item, notesdir) { + const path = `${notesdir}/${item.year}/${item.month}` + if(!existsSync(`${notesdir}/${item.year}`)) mkdirSync(`${notesdir}/${item.year}`) if(!existsSync(path)) mkdirSync(path) const mddata = `--- source: "${item.url}" context: "${item.context}" title: "${item.title}" -date: "${item.year}-${item.month}-${item.day}T${strpad(item.date.getHours())}:${strpad(item.date.getMinutes())}:${strpad(item.date.getSeconds())}" +date: "${item.year}-${item.month}-${item.day}T${item.date.format("HH:mm:ss")}" --- ${item.content} @@ -39,14 +35,17 @@ ${item.content} // opts: // notesdir = `${__dirname}/content/notes` // url = "https://chat.brainbaking.com/users/wouter/feed"; +// utcOffset = "+01:00" -async function parseMastoFeed(opts) { - const notesroot = await getFiles(opts.notesdir) +async function parseMastoFeed(options) { + const { notesdir, url, utcOffset = "+01:00" } = options + + const notesroot = await getFiles(notesdir) const notes = notesroot .filter(name => name.endsWith('.md')) .map(n => stripBeforeThirdSlash(n.replace('.md', ''))) - const buffer = await got(opts.url, { + const buffer = await got(url, { responseType: "buffer", resolveBodyOnly: true, timeout: 5000, @@ -56,10 +55,10 @@ async function parseMastoFeed(opts) { ignoreAttributes: false }) const items = root.feed.entry.map(item => { - const date = new Date(item.published) - const year = date.getFullYear() - const month = strpad(date.getMonth() + 1) - const day = strpad(date.getDate()) + const date = moment.utc(item.published).utcOffset(utcOffset) + const year = date.format("YYYY") + const month = date.format("MM") + const day = date.format("DD") // format: const context = item['thr:in-reply-to'] ? item['thr:in-reply-to']['@_ref'] : "" @@ -69,7 +68,7 @@ async function parseMastoFeed(opts) { url: item.id, // format: https://chat.brainbaking.com/objects/0707fd54-185d-4ee7-9204-be370d57663c context, id: stripBeforeLastSlash(item.id), - hash: `${day}h${date.getHours()}m${date.getMinutes()}s${date.getSeconds()}`, + hash: `${day}h${date.format("HH")}m${date.format("mm")}s${date.format("ss")}`, date, // format: 2021-03-02T16:18:46.658056Z year, month, @@ -77,7 +76,7 @@ async function parseMastoFeed(opts) { } }) .filter(itm => !notes.includes(`${itm.year}/${itm.month}/${itm.hash}`)) - .forEach(itm => convertAtomItemToMd(itm, opts)) + .forEach(itm => convertAtomItemToMd(itm, notesdir)) } module.exports = { diff --git a/test/masto-feed-parser.test.js b/test/masto-feed-parser.test.js index 59bf4cd..81bc35e 100644 --- a/test/masto-feed-parser.test.js +++ b/test/masto-feed-parser.test.js @@ -32,10 +32,11 @@ describe("mastodon feed parser tests", () => { test("parse creates correct MD structure", async () => { await parseMastoFeed({ url: "invalid", - notesdir: dumpdir + notesdir: dumpdir, + utcOffset: "+00:00" }) - const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m3s35.md`)).toString() + const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m03s35.md`)).toString() expect(actualMd).toMatchSnapshot() }) @@ -43,7 +44,8 @@ describe("mastodon feed parser tests", () => { //https://aus.social/users/aussocialadmin/statuses/105817435308293091 await parseMastoFeed({ url: "invalid", - notesdir: dumpdir + notesdir: dumpdir, + utcOffset: "+00:00" }) const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`) diff --git a/test/moment.test.js b/test/moment.test.js new file mode 100644 index 0000000..edd63ad --- /dev/null +++ b/test/moment.test.js @@ -0,0 +1,13 @@ + +// momentjs verification tests +const moment = require('moment') + +test('momentjs from UTC to UTC+1', () => { + + const date = moment + .utc("2021-03-02T16:13:27.921888Z") + .utcOffset("+01:00") + + expect(date.format("YYYY-MM-DD")).toEqual("2021-03-02") + expect(date.format("HH-mm-ss")).toEqual("17-13-27") +}) diff --git a/yarn.lock b/yarn.lock index d6029ca..e8452f6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2946,6 +2946,7 @@ fsevents@^2.1.2: howlongtobeat: ^1.3.1 jest: ^26.6.3 lunr: ^2.3.9 + moment: ^2.29.1 parser-front-matter: ^1.6.4 languageName: unknown linkType: soft @@ -3826,6 +3827,13 @@ fsevents@^2.1.2: languageName: node linkType: hard +"moment@npm:^2.29.1": + version: 2.29.1 + resolution: "moment@npm:2.29.1" + checksum: 86729013febf7160de5b93da69273dd304d674b0224f9544b3abd09a87671ddd2cdd57598261ce57588910d63747ffd5590965e83c790d8bf327083c0e0a06e0 + languageName: node + linkType: hard + "ms@npm:2.0.0": version: 2.0.0 resolution: "ms@npm:2.0.0"