lunr script added

This commit is contained in:
Wouter Groeneveld 2021-03-04 21:06:28 +01:00
parent c2020e20e3
commit 707a71d994
7 changed files with 111 additions and 12 deletions

16
src/file-utils.js Normal file
View File

@ -0,0 +1,16 @@
const fs = require('fs').promises;
const { resolve } = require('path');
// https://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
async function getFiles(dir) {
const dirents = await fs.readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
module.exports = {
getFiles
}

View File

@ -1,7 +1,9 @@
const { parseMastoFeed } = require('./masto-feed-parser')
const { widgetify } = require('./goodreads-widgetify.js')
const { buildIndex } = require('./lunr-index-builder.js')
module.exports = {
parseMastoFeed,
goodreadsWidgetify: widgetify
mastodonParseFeed: parseMastoFeed,
goodreadsWidgetify: widgetify,
lunrBuildIndex: buildIndex
};

48
src/lunr-index-builder.js Normal file
View File

@ -0,0 +1,48 @@
const fs = require('fs').promises;
const { getFiles } = require('./file-utils');
const { promisify } = require('util');
const frontMatterParser = require('parser-front-matter');
const parse = promisify(frontMatterParser.parse.bind(frontMatterParser));
async function loadPostsWithFrontMatter(postsDirectoryPath) {
const postNames = await getFiles(postsDirectoryPath);
const posts = await Promise.all(
// could be .DS_Store stuff found using recursive function above...
postNames.filter(name => name.endsWith('.md')).map(async fileName => {
const fileContent = await fs.readFile(fileName, 'utf8');
const {content, data} = await parse(fileContent);
return {
content: content.slice(0, 3000),
...data
};
})
);
return posts;
}
const lunrjs = require('lunr');
function makeIndex(posts) {
return lunrjs(function() {
this.ref('title');
this.field('title');
this.field('content');
this.field('tags');
posts.forEach(p => {
this.add(p);
});
});
}
async function run(contentDirs) {
const posts = await Promise.all(contentDirs.map(async (dir) => {
return await loadPostsWithFrontMatter(dir)
}))
return makeIndex(posts.flat());
}
module.exports = {
buildIndex: run
}

View File

@ -4,18 +4,9 @@ const fs = require('fs').promises;
const { writeFileSync, existsSync, mkdirSync } = require('fs');
const { resolve } = require('path');
const ent = require('ent')
const { getFiles } = require('./file-utils');
// https://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
async function getFiles(dir) {
const dirents = await fs.readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
function stripBeforeThirdSlash(str) {
const splitted = str.split('/')
return splitted.slice(splitted.length - 3).join('/')

View File

@ -0,0 +1,24 @@
const { buildIndex } = require('../src/lunr-index-builder')
let result = null
beforeEach(async () => {
result = await buildIndex([
`${__dirname}/postsstub1`,
`${__dirname}/postsstub2`])
})
test('lunr inverted index stuffed with loads of goodies from both dirs', async() => {
expect(result.invertedIndex.cool).not.toBe(undefined)
expect(result.invertedIndex.gravediggaz).not.toBe(undefined)
expect(result.invertedIndex.wu).not.toBe(undefined)
expect(result.invertedIndex.tang).not.toBe(undefined)
expect(result.invertedIndex.east).not.toBe(undefined)
expect(result.invertedIndex.side).not.toBe(undefined)
})
test('lunr index builder fields are title, content, tags', async () => {
// Do not forget to add JSON.Stringify() when calling this in production
expect(result.fields).toEqual(["title", "content", "tags"])
})

View File

@ -0,0 +1,8 @@
---
title: some cool article
date: 2021-03-02
---
Hi gayz whas goin' on gravediggaz style!
kkthxxbbye

View File

@ -0,0 +1,10 @@
---
title: another cool article east-side
date: 2021-03-03
---
It's just a hobby that I picked up in the lobby
---
The W, Wu-tang Clan yoo