save webmentions in separate domain folders, delete olders if requried

.gitignore

@@ -1,4 +1,6 @@
 
+data/*
+
 .yarn/*
 !.yarn/patches
 !.yarn/plugins

src/config.js

@@ -1,12 +1,30 @@
 
+const { existsSync, mkdirSync } = require('fs')
+
+const allowedWebmentionSources = [
+	"brainbaking.com",
+	"jefklakscodex.com"
+]
+
+function setupDataDirs() {
+	allowedWebmentionSources.forEach(domain => {
+		const dir = `data/${domain}`
+		console.log(` -- configured for ${domain}`)
+		if(!existsSync(dir)) {
+			mkdirSync(dir, {
+				recursive: true
+			})
+		}
+	})
+}
+
+
 module.exports = {
 	port: 4000,
 	host: "localhost",
 
 	utcOffset: 60,
 
-	allowedWebmentionSources: [
-		"brainbaking.com",
-		"jefklakscodex.com"
-	]
+	allowedWebmentionSources,
+	setupDataDirs
 }

src/serve.js

@@ -27,10 +27,10 @@ app.use(bodyParser({
    urlencoded: true
 }));
 
-
 // route docs: https://github.com/koajs/router/blob/HEAD/API.md#module_koa-router--Router+get%7Cput%7Cpost%7Cpatch%7Cdelete%7Cdel
 require("./webmention/route").route(router);
 const config = require("./config");
+config.setupDataDirs();
 
 app.use(router.routes()).use(router.allowedMethods());
 

src/webmention/receive.js

@@ -52,10 +52,18 @@ async function isValidTargetUrl(target) {
 	return false
 }
 
-async function saveWebmentionToDisk(source, target, mentiondata) {
-	console.log(`source=${source},target=${target}`)
+function asPath(source, target) {
 	const filename = md5(`source=${source},target=${target}`)
-	await fsp.writeFile(`data/${filename}.json`, mentiondata, 'utf-8')
+	const domain = config.allowedWebmentionSources.find(d => target.indexOf(d) >= 0)
+	return `data/${domain}/${filename}.json`
+}
+
+async function deletePossibleOlderWebmention(source, target) {
+	await fsp.unlink(asPath(source, target))
+}
+
+async function saveWebmentionToDisk(source, target, mentiondata) {
+	await fsp.writeFile(asPath(source, target), mentiondata, 'utf-8')
 }
 
 function publishedNow() {
@@ -121,6 +129,7 @@ async function receive(body) {
 		src = await got(body.source)
 	} catch(unknownSource) {
 		console.log(` ABORT: invalid source url: ` + unknownSource)
+		await deletePossibleOlderWebmention(body.source, body.target)
 		return
 	}
 	await processSourceBody(src.body, body.source, body.target)

test/__mocks__/got.js

@@ -1,7 +1,8 @@
 const fs = require('fs').promises
 
 async function got(url) {
-	const body = await fs.readFile(`./test/__mocks__/${url}`, 'utf8')
+	const relativeUrl = url.replace('https://brainbaking.com/', '')
+	const body = await fs.readFile(`./test/__mocks__/${relativeUrl}`, 'utf8')
 	return {
 		body
 	}

test/__mocks__/valid-indieweb-source-with-summary.html

@@ -282,7 +282,7 @@
         This is cool, this is a summary!
       </p>
       <div class="e-content content toot">
-        <p>This is cool, I just found out about <a href="valid-indieweb-target.html">valid indieweb target</a> - so cool</p>
+        <p>This is cool, I just found out about <a href="https://brainbaking.com/valid-indieweb-target.html">valid indieweb target</a> - so cool</p>
 
       </div>
 

test/__mocks__/valid-indieweb-source.html

@@ -279,7 +279,7 @@
 			</a>
 		</div>
 	    <div class="e-content content toot">
-	  		<p>This is cool, I just found out about <a href="valid-indieweb-target.html">valid indieweb target</a> - so cool</p>
+	  		<p>This is cool, I just found out about <a href="https://brainbaking.com/valid-indieweb-target.html">valid indieweb target</a> - so cool</p>
 
 	  	</div>
 

test/__mocks__/valid-nonindieweb-source.html

@@ -385,7 +385,7 @@
         <hr/>
         </header>
 
-        <p>Hey look at me i'm mentioning <a href="valid-indieweb-target.html">this cool site</a>!</p>
+        <p>Hey look at me i'm mentioning <a href="https://brainbaking.com/valid-indieweb-target.html">this cool site</a>!</p>
 
         <p>I&rsquo;ve been playing a lot of retro and more modern hack and slash games lately: from the more near-perfect D2 clone <a href="/articles/reviews/torchlight">Torchlight</a>, the more unique <a href="/articles/reviews/sacred">Sacred</a> and charming <a href="/articles/reviews/nox">Nox</a> to the boring <a href="/articles/reviews/dungeon-siege">Dungeon Siege</a> and too easy <a href="/articles/reviews/diablo-3">Diablo 3</a>. And then I found someone (my brother-in-law, also a hardcore D2 fan) to replay Diablo II and Lord of Destruction with, providing the perfect opportunity for a retrospective to see if Blizard North&rsquo;s smash hit still holds up compared to more recent hack &amp; slash revisions. The entire genre would not exist if it were not for this early 2000s game!</p>
 <h2 id="1-world-design">1. World Design</h2>

test/webmention/receive-process.test.js

@@ -6,7 +6,7 @@ const { rmdir } = require('./../utils')
 jest.mock('got');
 const md5 = require('md5')
 const { receive } = require('../../src/webmention/receive')
-const dumpdir = 'data'
+const dumpdir = 'data/brainbaking.com'
 const MockDate = require('mockdate')
 
 describe("receive webmention process tests happy path", () => {
@@ -20,14 +20,18 @@ describe("receive webmention process tests happy path", () => {
 		MockDate.set('2020-01-01')
 	})
 
-	test("receive saves a JSON file of indieweb-metadata if all is valid", async () => {
-		await receive({
-			source: "valid-indieweb-source.html",
-			target: "valid-indieweb-target.html"
-		})
+	function asFilename(body) {
+		return `${dumpdir}/` + md5(`source=${body.source},target=${body.target}`)
+	}
 
-		const filename = md5(`source=valid-indieweb-source.html,target=valid-indieweb-target.html`)
-		const result = await fsp.readFile(`data/${filename}.json`, 'utf-8')
+	test("receive saves a JSON file of indieweb-metadata if all is valid", async () => {
+		const body = {
+			source: "https://brainbaking.com/valid-indieweb-source.html",
+			target: "https://brainbaking.com/valid-indieweb-target.html"
+		}
+		await receive(body)
+
+		const result = await fsp.readFile(`${asFilename(body)}.json`, 'utf-8')
 		const data = JSON.parse(result)
 
 		expect(data).toEqual({
@@ -36,38 +40,74 @@ describe("receive webmention process tests happy path", () => {
 				picture: "https://brainbaking.com//img/avatar.jpg"
 			},
 			content: "This is cool, I just found out about valid indieweb target - so cool...",
-			source: "valid-indieweb-source.html",
-			target: "valid-indieweb-target.html",
+			source: body.source,
+			target: body.target,
+			published: "2021-03-06T12:41:00"
+		})
+	})
+
+	test("receive saves a JSON file of indieweb-metadata with summary as content if present", async () => {
+		const body = {
+			source: "https://brainbaking.com/valid-indieweb-source-with-summary.html",
+			target: "https://brainbaking.com/valid-indieweb-target.html"
+		}
+		await receive(body)
+
+		const result = await fsp.readFile(`${asFilename(body)}.json`, 'utf-8')
+		const data = JSON.parse(result)
+
+		expect(data).toEqual({
+			author: {
+				name: "Wouter Groeneveld",
+				picture: "https://brainbaking.com//img/avatar.jpg"
+			},
+			content: "This is cool, this is a summary!",
+			source: body.source,
+			target: body.target,
 			published: "2021-03-06T12:41:00"
 		})
 	})
 
 	test("receive saves a JSON file of non-indieweb-data such as title if all is valid", async () => {
-		await receive({
-			source: "valid-nonindieweb-source.html",
-			target: "valid-indieweb-target.html"
-		})
+		const body = {
+			source: "https://brainbaking.com/valid-nonindieweb-source.html",
+			target: "https://brainbaking.com/valid-indieweb-target.html"
+		}
+		await receive(body)
 
-		const filename = md5(`source=valid-nonindieweb-source.html,target=valid-indieweb-target.html`)
-		const result = await fsp.readFile(`data/${filename}.json`, 'utf-8')
+		const result = await fsp.readFile(`${asFilename(body)}.json`, 'utf-8')
 		const data = JSON.parse(result)
 
 		expect(data).toEqual({
 			author: {
-				name: "valid-nonindieweb-source.html",
+				name: "https://brainbaking.com/valid-nonindieweb-source.html",
 			},
 			content: "Diablo 2 Twenty Years Later: A Retrospective | Jefklaks Codex",
-			source: "valid-nonindieweb-source.html",
-			target: "valid-indieweb-target.html",
+			source: body.source,
+			target: body.target,
 			published: "2020-01-01T01:00:00"
 		})
 	})
 
  	test("receive a target that does not point to the source does nothing", async () => {
-		await receive({
-			source: "valid-indieweb-source.html",
-			target: "valid-indieweb-source.html"
-		})
+ 		const body = {
+			source: "https://brainbaking.com/valid-indieweb-source.html",
+			target: "https://brainbaking.com/valid-indieweb-source.html"
+		}
+		await receive(body)
+
+		const data = fs.readdirSync(dumpdir)
+		expect(data.length).toBe(0)
+	})
+
+ 	test("receive a source that does not exist should also delete older webmention files", async () => {
+ 		const body = {
+			source: "https://wubanga2001.boom/lolz",
+			target: "https://brainbaking.com/valid-indieweb-source.html"
+		}
+
+		await fsp.writeFile(`${asFilename(body)}.json`, JSON.stringify({ lolz: "aha" }), 'utf-8')
+		await receive(body)
 
 		const data = fs.readdirSync(dumpdir)
 		expect(data.length).toBe(0)