migrate jamstack scripts to node module, use yarn
parent
d3f1682f78
commit
c0e789bcb9
|
|
@ -18,4 +18,11 @@ books/**/*.toc
|
|||
books/**/*.pdf
|
||||
books/**/*.lot
|
||||
books/**/*.aux
|
||||
books/**/boek.tex
|
||||
books/**/boek.tex
|
||||
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,3 @@
|
|||
yarnPath: .yarn/releases/yarn-2.4.1.cjs
|
||||
# needed for hugo babel pipelining
|
||||
nodeLinker: node-modules
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
module.exports = function (api) {
|
||||
api.cache(true);
|
||||
|
||||
const presets = [ '@babel/preset-env' ];
|
||||
const presets = [
|
||||
require('@babel/preset-env')
|
||||
];
|
||||
const plugins = [ ];
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
const fs = require('fs').promises;
|
||||
const { resolve } = require('path');
|
||||
|
||||
const {promisify} = require('util');
|
||||
const frontMatterParser = require('parser-front-matter');
|
||||
|
||||
const parse = promisify(frontMatterParser.parse.bind(frontMatterParser));
|
||||
|
||||
// https://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
|
||||
async function getFiles(dir) {
|
||||
const dirents = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files = await Promise.all(dirents.map((dirent) => {
|
||||
const res = resolve(dir, dirent.name);
|
||||
return dirent.isDirectory() ? getFiles(res) : res;
|
||||
}));
|
||||
return Array.prototype.concat(...files);
|
||||
}
|
||||
|
||||
async function loadPostsWithFrontMatter(postsDirectoryPath) {
|
||||
const postNames = await getFiles(postsDirectoryPath);
|
||||
const posts = await Promise.all(
|
||||
// could be .DS_Store stuff found using recursive function above...
|
||||
postNames.filter(name => name.endsWith('.md')).map(async fileName => {
|
||||
const fileContent = await fs.readFile(fileName, 'utf8');
|
||||
const {content, data} = await parse(fileContent);
|
||||
return {
|
||||
content: content.slice(0, 3000),
|
||||
...data
|
||||
};
|
||||
})
|
||||
);
|
||||
return posts;
|
||||
}
|
||||
|
||||
const lunrjs = require('lunr');
|
||||
|
||||
function makeIndex(posts) {
|
||||
return lunrjs(function() {
|
||||
this.ref('title');
|
||||
this.field('title');
|
||||
this.field('content');
|
||||
this.field('tags');
|
||||
posts.forEach(p => {
|
||||
this.add(p);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const posts = await loadPostsWithFrontMatter(`${__dirname}/content/post`);
|
||||
const notes = await loadPostsWithFrontMatter(`${__dirname}/content/notes`);
|
||||
const index = makeIndex(posts.concat(notes));
|
||||
console.log(JSON.stringify(index));
|
||||
}
|
||||
|
||||
run()
|
||||
.then(() => process.exit(0))
|
||||
.catch(error => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
source: "https://chat.brainbaking.com/objects/3d01c975-bdf7-430d-b65d-069362a937bf"
|
||||
context: "https://bsd.network/users/rubenerd/statuses/105831716720720259"
|
||||
title: "@rubenerd out of curiosity: are both synced in any way? If not, are you sure the migraine isnt fr..."
|
||||
date: "2021-03-04T14:52:55"
|
||||
---
|
||||
|
||||
<span class="h-card"><a class="u-url mention" data-user="A4fIbZoZ0HNJPHV4Vs" href="https://bsd.network/@rubenerd" rel="ugc">@<span>rubenerd</span></a></span> out of curiosity: are both synced in any way? If not, are you sure the migraine isnt from checking both walls? 😇
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
source: "https://chat.brainbaking.com/objects/9957a699-1cdb-43c6-aea7-412891b03a83"
|
||||
context: ""
|
||||
title: "I spent some time with Yarn/Jest/ES2019, it felt good to be programming again. I am developing a ..."
|
||||
date: "2021-03-05T08:51:41"
|
||||
---
|
||||
|
||||
I spent some time with Yarn/Jest/ES2019, it felt good to be programming again. I am developing a serious JavaScript FOMO since switching from the software engineering industry to academia, and I'm not quite sure if I like that...
|
||||
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
title: Half-Baked Ideas
|
||||
type: note
|
||||
---
|
||||
|
||||
> A half-baked idea a day keeps the doctor away. <span>Wouter</span>
|
||||
|
||||
These are fleeting, _half-baked_ thoughts, that may or may not get fully baked into fleshed out [blog posts](/post). The notes below are also syndicated to other platforms, such as <svg class='icon icon-text'><use xlink:href='#discuss'></use></svg>[Mastodon](https://chat.brainbaking.com/@wouter). Enjoy reading my ramblings!
|
||||
|
||||
Not finding what you're looking for? [Browse the archives](/archives).<br/>
|
||||
Want to stay up to date? [Subscribe to the <svg class='icon icon-small' width='16' height='16'><use xlink:href='#news'></use></svg>Notes Feed](/notes/index.xml).<br/>
|
||||
Interested more substantial thoughts? [Read the Freshly Baked blog](/post).
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
const https = require('https')
|
||||
|
||||
|
||||
// WHY? Because including this thing comes with free cookies...
|
||||
const url = "https://www.goodreads.com/review/grid_widget/5451893.Wouter's%20bookshelf:%20read?cover_size=medium&hide_link=&hide_title=&num_books=12&order=d&shelf=read&sort=date_added&widget_id=1496758344"
|
||||
|
||||
const replaceLowresWithHiresImages = (data) => {
|
||||
return data.replace(/_SX[0-9]+_(SY[0-9]+_)*.jpg/g, "_S400_.jpg")
|
||||
}
|
||||
|
||||
https.get(url, (resp) => {
|
||||
let data = '';
|
||||
|
||||
resp.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
resp.on('end', () => {
|
||||
console.log(replaceLowresWithHiresImages(data))
|
||||
});
|
||||
})
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
|
||||
const { mastodon, goodreads, lunr } = require('jam-my-stack');
|
||||
const fsp = require('fs').promises;
|
||||
|
||||
|
||||
(async function() {
|
||||
// 1. parse Mastodon RSS feed
|
||||
console.log("1. parsing Mastodon RSS feed...")
|
||||
await mastodon.parseFeed({
|
||||
notesdir: `${__dirname}/content/notes`,
|
||||
url: "https://chat.brainbaking.com/users/wouter/feed"
|
||||
})
|
||||
|
||||
// 2. update goodreads JS widget
|
||||
console.log("2. Updating Goodreads Widget...")
|
||||
const widget = await goodreads.createWidget("https://www.goodreads.com/review/grid_widget/5451893.Wouter's%20bookshelf:%20read?cover_size=medium&hide_link=&hide_title=&num_books=12&order=d&shelf=read&sort=date_added&widget_id=1496758344")
|
||||
await fsp.writeFile(`${__dirname}/static/js/goodreads.js`, widget, 'utf-8')
|
||||
|
||||
// 3. build Lunr index
|
||||
console.log("3. Building lunr search index...")
|
||||
const index = await lunr.buildIndex([
|
||||
`${__dirname}/content/post`,
|
||||
`${__dirname}/content/notes`])
|
||||
await fsp.writeFile(`${__dirname}/static/js/brainbaking-post.json`, JSON.stringify(index), 'utf-8')
|
||||
|
||||
|
||||
console.log("-- all done!")
|
||||
})()
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
const got = require("got");
|
||||
const parser = require("fast-xml-parser");
|
||||
const fs = require('fs').promises;
|
||||
const { writeFileSync, existsSync, mkdirSync } = require('fs');
|
||||
const { resolve } = require('path');
|
||||
const ent = require('ent')
|
||||
|
||||
const notesdir = `${__dirname}/content/notes`
|
||||
const url = "https://chat.brainbaking.com/users/wouter/feed";
|
||||
|
||||
// https://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
|
||||
async function getFiles(dir) {
|
||||
const dirents = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files = await Promise.all(dirents.map((dirent) => {
|
||||
const res = resolve(dir, dirent.name);
|
||||
return dirent.isDirectory() ? getFiles(res) : res;
|
||||
}));
|
||||
return Array.prototype.concat(...files);
|
||||
}
|
||||
|
||||
function stripBeforeThirdSlash(str) {
|
||||
const splitted = str.split('/')
|
||||
return splitted.slice(splitted.length - 3).join('/')
|
||||
}
|
||||
|
||||
function stripBeforeLastSlash(str) {
|
||||
return str.substring(str.lastIndexOf('/') + 1, str.length)
|
||||
}
|
||||
|
||||
function strpad(n) {
|
||||
return String("0" + n).slice(-2);
|
||||
}
|
||||
|
||||
function convertAtomItemToMd(item) {
|
||||
const path = `${notesdir}/${item.year}/${item.month}`
|
||||
if(!existsSync(`${notesdir}/${item.year}`)) mkdirSync(`${notesdir}/${item.year}`)
|
||||
if(!existsSync(path)) mkdirSync(path)
|
||||
|
||||
const mddata = `---
|
||||
source: "${item.url}"
|
||||
context: "${item.context}"
|
||||
title: "${item.title}"
|
||||
date: "${item.year}-${item.month}-${item.day}T${strpad(item.date.getHours())}:${strpad(item.date.getMinutes())}:${strpad(item.date.getSeconds())}"
|
||||
---
|
||||
|
||||
${item.content}
|
||||
`
|
||||
|
||||
writeFileSync(`${path}/${item.hash}.md`, mddata, 'utf-8')
|
||||
}
|
||||
|
||||
(async function main() {
|
||||
const notesroot = await getFiles(notesdir)
|
||||
const notes = notesroot
|
||||
.filter(name => name.endsWith('.md'))
|
||||
.map(n => stripBeforeThirdSlash(n.replace('.md', '')))
|
||||
|
||||
const buffer = await got(url, {
|
||||
responseType: "buffer",
|
||||
resolveBodyOnly: true,
|
||||
timeout: 5000,
|
||||
retry: 5
|
||||
});
|
||||
const root = parser.parse(buffer.toString(), {
|
||||
ignoreAttributes: false
|
||||
})
|
||||
const items = root.feed.entry.map(item => {
|
||||
const date = new Date(item.published)
|
||||
const year = date.getFullYear()
|
||||
const month = strpad(date.getMonth() + 1)
|
||||
const day = strpad(date.getDate())
|
||||
// format: <thr:in-reply-to ref='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793' href='https://social.linux.pizza/users/StampedingLonghorn/statuses/105821099684887793'/>
|
||||
const context = item['thr:in-reply-to'] ? item['thr:in-reply-to']['@_ref'] : ""
|
||||
|
||||
return {
|
||||
title: ent.decode(item.title), // summary (cut-off) of content
|
||||
content: ent.decode(item.content['#text']), // format: <span class="h-card....
|
||||
url: item.id, // format: https://chat.brainbaking.com/objects/0707fd54-185d-4ee7-9204-be370d57663c
|
||||
context,
|
||||
id: stripBeforeLastSlash(item.id),
|
||||
hash: `${day}h${date.getHours()}m${date.getMinutes()}s${date.getSeconds()}`,
|
||||
date, // format: 2021-03-02T16:18:46.658056Z
|
||||
year,
|
||||
month,
|
||||
day
|
||||
}
|
||||
})
|
||||
.filter(itm => !notes.includes(`${itm.year}/${itm.month}/${itm.hash}`))
|
||||
.forEach(convertAtomItemToMd)
|
||||
|
||||
console.log('Done!')
|
||||
})()
|
||||
File diff suppressed because it is too large
Load Diff
19
package.json
19
package.json
|
|
@ -2,34 +2,29 @@
|
|||
"name": "brainbaking",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "goodreads-fetch.js",
|
||||
"main": "jamstack-init.js",
|
||||
"directories": {
|
||||
"doc": "docs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.8.4",
|
||||
"@babel/core": "^7.9.6",
|
||||
"@babel/preset-env": "^7.10.1",
|
||||
"lunr": "^2.3.6",
|
||||
"parser-front-matter": "^1.6.4"
|
||||
"@babel/preset-env": "^7.10.1"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "/usr/local/bin/hugo",
|
||||
"install": "node masto-feed-parser.js && node goodreads-fetch.js > static/js/goodreads.js && node build-lunr-index.js > static/js/brainbaking-post.json"
|
||||
"jamstack": "node jamstack-init.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/wgroeneveld/brainbaking.git"
|
||||
},
|
||||
"author": "",
|
||||
"license": "",
|
||||
"author": "Wouter Groeneveld",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://github.com/wgroeneveld/brainbaking/issues"
|
||||
},
|
||||
"homepage": "https://github.com/wgroeneveld/brainbaking#readme",
|
||||
"homepage": "https://brainbaking.com",
|
||||
"dependencies": {
|
||||
"ent": "^2.2.0",
|
||||
"fast-xml-parser": "^3.18.0",
|
||||
"got": "^11.8.2"
|
||||
"jam-my-stack": "^1.0.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue