use moment.js insted of parsing dates yourself...
This commit is contained in:
parent
542302f171
commit
896b965879
|
@ -43,6 +43,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
|
||||||
["howlongtobeat", "npm:1.3.1"],
|
["howlongtobeat", "npm:1.3.1"],
|
||||||
["jest", "npm:26.6.3"],
|
["jest", "npm:26.6.3"],
|
||||||
["lunr", "npm:2.3.9"],
|
["lunr", "npm:2.3.9"],
|
||||||
|
["moment", "npm:2.29.1"],
|
||||||
["parser-front-matter", "npm:1.6.4"]
|
["parser-front-matter", "npm:1.6.4"]
|
||||||
],
|
],
|
||||||
"linkType": "SOFT",
|
"linkType": "SOFT",
|
||||||
|
@ -3523,6 +3524,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
|
||||||
["howlongtobeat", "npm:1.3.1"],
|
["howlongtobeat", "npm:1.3.1"],
|
||||||
["jest", "npm:26.6.3"],
|
["jest", "npm:26.6.3"],
|
||||||
["lunr", "npm:2.3.9"],
|
["lunr", "npm:2.3.9"],
|
||||||
|
["moment", "npm:2.29.1"],
|
||||||
["parser-front-matter", "npm:1.6.4"]
|
["parser-front-matter", "npm:1.6.4"]
|
||||||
],
|
],
|
||||||
"linkType": "SOFT",
|
"linkType": "SOFT",
|
||||||
|
@ -4505,6 +4507,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
|
||||||
"linkType": "HARD",
|
"linkType": "HARD",
|
||||||
}]
|
}]
|
||||||
]],
|
]],
|
||||||
|
["moment", [
|
||||||
|
["npm:2.29.1", {
|
||||||
|
"packageLocation": "./.yarn/cache/moment-npm-2.29.1-787d9fdafd-86729013fe.zip/node_modules/moment/",
|
||||||
|
"packageDependencies": [
|
||||||
|
["moment", "npm:2.29.1"]
|
||||||
|
],
|
||||||
|
"linkType": "HARD",
|
||||||
|
}]
|
||||||
|
]],
|
||||||
["ms", [
|
["ms", [
|
||||||
["npm:2.0.0", {
|
["npm:2.0.0", {
|
||||||
"packageLocation": "./.yarn/cache/ms-npm-2.0.0-9e1101a471-1a230340cc.zip/node_modules/ms/",
|
"packageLocation": "./.yarn/cache/ms-npm-2.0.0-9e1101a471-1a230340cc.zip/node_modules/ms/",
|
||||||
|
|
|
@ -26,10 +26,15 @@ Usage example:
|
||||||
```js
|
```js
|
||||||
await mastodon.parseFeed({
|
await mastodon.parseFeed({
|
||||||
notesdir: `${__dirname}/content/notes`,
|
notesdir: `${__dirname}/content/notes`,
|
||||||
url: "https://chat.brainbaking.com/users/wouter/feed"
|
url: "https://chat.brainbaking.com/users/wouter/feed",
|
||||||
|
utcOffset: "+01:00"
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Default values:
|
||||||
|
|
||||||
|
- `utcOffset`: `"+01:00"` (because that's where I am!)
|
||||||
|
|
||||||
Note that this **does not** delete the notes dir with every call. It simply checks if there isn't already a file with the same name (based on the publication date), and adds one if not.
|
Note that this **does not** delete the notes dir with every call. It simply checks if there isn't already a file with the same name (based on the publication date), and adds one if not.
|
||||||
|
|
||||||
Example feed entry:
|
Example feed entry:
|
||||||
|
@ -56,7 +61,7 @@ Example feed entry:
|
||||||
</entry>
|
</entry>
|
||||||
```
|
```
|
||||||
|
|
||||||
This generates the file `01h20m3s35.md` (it uses your local timezone based on the `<published/>` tag - mine is GMT+1) with contents:
|
This generates the file `01h20m03s35.md` (it assumes UTC times in the feed and adjusts according to specified `utcOffset`, such as GMT+1 in this example), with contents:
|
||||||
|
|
||||||
```md
|
```md
|
||||||
---
|
---
|
||||||
|
|
|
@ -1 +1,3 @@
|
||||||
process.env.TZ = 'GMT';
|
|
||||||
|
// doing this on purpose to force test and implementation to use corret moment.js parsing
|
||||||
|
process.env.TZ = 'GMT+6';
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "jam-my-stack",
|
"name": "jam-my-stack",
|
||||||
"version": "1.0.3",
|
"version": "1.0.4",
|
||||||
"repository": {
|
"repository": {
|
||||||
"url": "https://github.com/wgroeneveld/jam-my-stack",
|
"url": "https://github.com/wgroeneveld/jam-my-stack",
|
||||||
"type": "git"
|
"type": "git"
|
||||||
|
@ -17,6 +17,7 @@
|
||||||
"got": "^11.8.2",
|
"got": "^11.8.2",
|
||||||
"howlongtobeat": "^1.3.1",
|
"howlongtobeat": "^1.3.1",
|
||||||
"lunr": "^2.3.9",
|
"lunr": "^2.3.9",
|
||||||
|
"moment": "^2.29.1",
|
||||||
"parser-front-matter": "^1.6.4"
|
"parser-front-matter": "^1.6.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
@ -3,7 +3,7 @@ const parser = require("fast-xml-parser");
|
||||||
const { writeFileSync, existsSync, mkdirSync } = require('fs');
|
const { writeFileSync, existsSync, mkdirSync } = require('fs');
|
||||||
const ent = require('ent')
|
const ent = require('ent')
|
||||||
const { getFiles } = require('./file-utils');
|
const { getFiles } = require('./file-utils');
|
||||||
|
const moment = require('moment')
|
||||||
|
|
||||||
function stripBeforeThirdSlash(str) {
|
function stripBeforeThirdSlash(str) {
|
||||||
const splitted = str.split('/')
|
const splitted = str.split('/')
|
||||||
|
@ -14,20 +14,16 @@ function stripBeforeLastSlash(str) {
|
||||||
return str.substring(str.lastIndexOf('/') + 1, str.length)
|
return str.substring(str.lastIndexOf('/') + 1, str.length)
|
||||||
}
|
}
|
||||||
|
|
||||||
function strpad(n) {
|
function convertAtomItemToMd(item, notesdir) {
|
||||||
return String("0" + n).slice(-2);
|
const path = `${notesdir}/${item.year}/${item.month}`
|
||||||
}
|
if(!existsSync(`${notesdir}/${item.year}`)) mkdirSync(`${notesdir}/${item.year}`)
|
||||||
|
|
||||||
function convertAtomItemToMd(item, opts) {
|
|
||||||
const path = `${opts.notesdir}/${item.year}/${item.month}`
|
|
||||||
if(!existsSync(`${opts.notesdir}/${item.year}`)) mkdirSync(`${opts.notesdir}/${item.year}`)
|
|
||||||
if(!existsSync(path)) mkdirSync(path)
|
if(!existsSync(path)) mkdirSync(path)
|
||||||
|
|
||||||
const mddata = `---
|
const mddata = `---
|
||||||
source: "${item.url}"
|
source: "${item.url}"
|
||||||
context: "${item.context}"
|
context: "${item.context}"
|
||||||
title: "${item.title}"
|
title: "${item.title}"
|
||||||
date: "${item.year}-${item.month}-${item.day}T${strpad(item.date.getHours())}:${strpad(item.date.getMinutes())}:${strpad(item.date.getSeconds())}"
|
date: "${item.year}-${item.month}-${item.day}T${item.date.format("HH:mm:ss")}"
|
||||||
---
|
---
|
||||||
|
|
||||||
${item.content}
|
${item.content}
|
||||||
|
@ -39,14 +35,17 @@ ${item.content}
|
||||||
// opts:
|
// opts:
|
||||||
// notesdir = `${__dirname}/content/notes`
|
// notesdir = `${__dirname}/content/notes`
|
||||||
// url = "https://chat.brainbaking.com/users/wouter/feed";
|
// url = "https://chat.brainbaking.com/users/wouter/feed";
|
||||||
|
// utcOffset = "+01:00"
|
||||||
|
|
||||||
async function parseMastoFeed(opts) {
|
async function parseMastoFeed(options) {
|
||||||
const notesroot = await getFiles(opts.notesdir)
|
const { notesdir, url, utcOffset = "+01:00" } = options
|
||||||
|
|
||||||
|
const notesroot = await getFiles(notesdir)
|
||||||
const notes = notesroot
|
const notes = notesroot
|
||||||
.filter(name => name.endsWith('.md'))
|
.filter(name => name.endsWith('.md'))
|
||||||
.map(n => stripBeforeThirdSlash(n.replace('.md', '')))
|
.map(n => stripBeforeThirdSlash(n.replace('.md', '')))
|
||||||
|
|
||||||
const buffer = await got(opts.url, {
|
const buffer = await got(url, {
|
||||||
responseType: "buffer",
|
responseType: "buffer",
|
||||||
resolveBodyOnly: true,
|
resolveBodyOnly: true,
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
|
@ -56,10 +55,10 @@ async function parseMastoFeed(opts) {
|
||||||
ignoreAttributes: false
|
ignoreAttributes: false
|
||||||
})
|
})
|
||||||
const items = root.feed.entry.map(item => {
|
const items = root.feed.entry.map(item => {
|
||||||
const date = new Date(item.published)
|
const date = moment.utc(item.published).utcOffset(utcOffset)
|
||||||
const year = date.getFullYear()
|
const year = date.format("YYYY")
|
||||||
const month = strpad(date.getMonth() + 1)
|
const month = date.format("MM")
|
||||||
const day = strpad(date.getDate())
|
const day = date.format("DD")
|
||||||
// format: <thr:in-reply-to ref='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793' href='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793'/>
|
// format: <thr:in-reply-to ref='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793' href='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793'/>
|
||||||
const context = item['thr:in-reply-to'] ? item['thr:in-reply-to']['@_ref'] : ""
|
const context = item['thr:in-reply-to'] ? item['thr:in-reply-to']['@_ref'] : ""
|
||||||
|
|
||||||
|
@ -69,7 +68,7 @@ async function parseMastoFeed(opts) {
|
||||||
url: item.id, // format: https://chat.brainbaking.com/objects/0707fd54-185d-4ee7-9204-be370d57663c
|
url: item.id, // format: https://chat.brainbaking.com/objects/0707fd54-185d-4ee7-9204-be370d57663c
|
||||||
context,
|
context,
|
||||||
id: stripBeforeLastSlash(item.id),
|
id: stripBeforeLastSlash(item.id),
|
||||||
hash: `${day}h${date.getHours()}m${date.getMinutes()}s${date.getSeconds()}`,
|
hash: `${day}h${date.format("HH")}m${date.format("mm")}s${date.format("ss")}`,
|
||||||
date, // format: 2021-03-02T16:18:46.658056Z
|
date, // format: 2021-03-02T16:18:46.658056Z
|
||||||
year,
|
year,
|
||||||
month,
|
month,
|
||||||
|
@ -77,7 +76,7 @@ async function parseMastoFeed(opts) {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.filter(itm => !notes.includes(`${itm.year}/${itm.month}/${itm.hash}`))
|
.filter(itm => !notes.includes(`${itm.year}/${itm.month}/${itm.hash}`))
|
||||||
.forEach(itm => convertAtomItemToMd(itm, opts))
|
.forEach(itm => convertAtomItemToMd(itm, notesdir))
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -32,10 +32,11 @@ describe("mastodon feed parser tests", () => {
|
||||||
test("parse creates correct MD structure", async () => {
|
test("parse creates correct MD structure", async () => {
|
||||||
await parseMastoFeed({
|
await parseMastoFeed({
|
||||||
url: "invalid",
|
url: "invalid",
|
||||||
notesdir: dumpdir
|
notesdir: dumpdir,
|
||||||
|
utcOffset: "+00:00"
|
||||||
})
|
})
|
||||||
|
|
||||||
const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m3s35.md`)).toString()
|
const actualMd = (await fsp.readFile(`${dumpdir}/2021/03/01h19m03s35.md`)).toString()
|
||||||
expect(actualMd).toMatchSnapshot()
|
expect(actualMd).toMatchSnapshot()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -43,7 +44,8 @@ describe("mastodon feed parser tests", () => {
|
||||||
//https://aus.social/users/aussocialadmin/statuses/105817435308293091
|
//https://aus.social/users/aussocialadmin/statuses/105817435308293091
|
||||||
await parseMastoFeed({
|
await parseMastoFeed({
|
||||||
url: "invalid",
|
url: "invalid",
|
||||||
notesdir: dumpdir
|
notesdir: dumpdir,
|
||||||
|
utcOffset: "+00:00"
|
||||||
})
|
})
|
||||||
|
|
||||||
const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)
|
const actualMd = await fsp.readFile(`${dumpdir}/2021/03/02h16m18s46.md`)
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
|
||||||
|
// momentjs verification tests
|
||||||
|
const moment = require('moment')
|
||||||
|
|
||||||
|
test('momentjs from UTC to UTC+1', () => {
|
||||||
|
|
||||||
|
const date = moment
|
||||||
|
.utc("2021-03-02T16:13:27.921888Z")
|
||||||
|
.utcOffset("+01:00")
|
||||||
|
|
||||||
|
expect(date.format("YYYY-MM-DD")).toEqual("2021-03-02")
|
||||||
|
expect(date.format("HH-mm-ss")).toEqual("17-13-27")
|
||||||
|
})
|
|
@ -2946,6 +2946,7 @@ fsevents@^2.1.2:
|
||||||
howlongtobeat: ^1.3.1
|
howlongtobeat: ^1.3.1
|
||||||
jest: ^26.6.3
|
jest: ^26.6.3
|
||||||
lunr: ^2.3.9
|
lunr: ^2.3.9
|
||||||
|
moment: ^2.29.1
|
||||||
parser-front-matter: ^1.6.4
|
parser-front-matter: ^1.6.4
|
||||||
languageName: unknown
|
languageName: unknown
|
||||||
linkType: soft
|
linkType: soft
|
||||||
|
@ -3826,6 +3827,13 @@ fsevents@^2.1.2:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"moment@npm:^2.29.1":
|
||||||
|
version: 2.29.1
|
||||||
|
resolution: "moment@npm:2.29.1"
|
||||||
|
checksum: 86729013febf7160de5b93da69273dd304d674b0224f9544b3abd09a87671ddd2cdd57598261ce57588910d63747ffd5590965e83c790d8bf327083c0e0a06e0
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"ms@npm:2.0.0":
|
"ms@npm:2.0.0":
|
||||||
version: 2.0.0
|
version: 2.0.0
|
||||||
resolution: "ms@npm:2.0.0"
|
resolution: "ms@npm:2.0.0"
|
||||||
|
|
Loading…
Reference in New Issue