add webmention getpicture functionality

.gitignore

@@ -9,3 +9,4 @@ dump
 !.yarn/versions
 
 *.sublime-workspace
+node_modules/

.yarnrc.yml

@@ -1 +1,2 @@
 yarnPath: ".yarn/releases/yarn-berry.cjs"
+nodeLinker: node-modules

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jam-my-stack",
-  "version": "1.0.22",
+  "version": "1.0.25",
   "repository": {
     "url": "https://github.com/wgroeneveld/jam-my-stack",
     "type": "git"
@@ -16,7 +16,7 @@
     "ejs": "^3.1.6",
     "ent": "^2.2.0",
     "fast-xml-parser": "^3.18.0",
-    "got": "^11.8.2",
+    "got": "11.8.3",
     "howlongtobeat": "^1.3.1",
     "imagemagick": "^0.1.3",
     "lunr": "^2.3.9",

src/index.js

@@ -6,6 +6,7 @@ const { thumbify } = require('./youtube/thumbify.js')
 
 const { getWebmentions } = require('./webmention/get.js')
 const { sendWebmentions } = require('./webmention/send.js')
+const { getPictures } = require('./webmention/pictures.js')
 
 module.exports = {
 	mastodon: {
@@ -25,6 +26,7 @@ module.exports = {
 	},
 	webmention: {
 		getWebmentions: getWebmentions,
-		send: sendWebmentions
+		send: sendWebmentions,
+		getPictures
 	}
 };

src/webmention/pictures.js

@@ -0,0 +1,45 @@
+const got = require("got");
+const { createWriteStream } = require("fs");
+const { pipeline } = require('stream/promises');
+const { existsSync } = require('fs');
+const fsp = require('fs').promises;
+const { dirname } = require('path');
+
+/**
+ * config example: {
+ * 		endpoint: "https://bla.com",
+ * 		directory: "dir/to/save/to",
+ * 		override: true, 	// optional
+ * 		extension: "jpg" 	// optional
+ * }
+ **/
+async function getPictures(webmentions, config) {
+	const pics = await Promise.all(
+		webmentions.filter(wm => wm.author && wm.author.picture).map(async wm => {
+			let fileName = `${config.directory}/${wm.author.picture}`
+			if(config.extension && !fileName.endsWith(config.extension)) {
+				fileName += `.${config.extension}`
+			}
+
+			if(config.override || !existsSync(fileName)) {
+				try {
+					await fsp.mkdir(dirname(fileName), { recursive: true }),
+					// retry API with streams is ridiculously complicated: https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md
+					await pipeline(
+						got.stream(`${config.endpoint}/${wm.author.picture}`),
+						createWriteStream(fileName)
+					)
+				} catch(err) {
+					console.log(` !! unable to download ${wm.author.picture}: ${err.message}`)
+					await fsp.unlink(fileName)
+				}
+			}
+			return fileName
+		})
+	)
+	return pics
+}
+
+module.exports = {
+	getPictures
+}

src/webmention/send.js

@@ -1,7 +1,5 @@
 
 const got = require('got')
-const fsp = require('fs').promises
-
 const dayjs = require('dayjs')
 
 async function sendWebmentions(domain, config) {

test/webmention/get-e2e.test.js

@@ -9,11 +9,14 @@ describe("webmention receive scenario test", () => {
 
 	// this tests against the real endpoint, meaning the token has to be correct.
 	test("getWebmentions fetches anything at all", async () => {
-		const result = await getWebmentions(domain, {
-			token: 'miauwkes',
-			endpoint: 'https://jam.brainbaking.com'			
-		})
-		//expect(result.length).toBeGreaterThan(-1)
+		try {
+			const result = await getWebmentions(domain, {
+				token: process.env.WEBMENTION_TOKEN,
+				endpoint: 'https://jam.brainbaking.com'			
+			})
+			//expect(result.length).toBeGreaterThan(-1)
+		} catch {
+		}
 	})
 
 })

test/webmention/pictures.test.js

@@ -0,0 +1,91 @@
+const fs = require('fs');
+const fsp = require('fs').promises;
+const { rmdir } = require('./../utils')
+const { getFiles } = require('../../src/file-utils');
+
+const dumpdir = `${__dirname}/dump`
+
+beforeEach(async () => {
+	if(fs.existsSync(dumpdir)) {
+		rmdir(dumpdir)
+	}
+	fs.mkdirSync(dumpdir)
+})
+
+jest.disableAutomock()
+jest.unmock('got')
+
+describe("webmention downloading of author pictures", () => {
+
+	const { getPictures } = require('./../../src/webmention/pictures')
+
+	test("getPictures does not write a file if error occurs", async () => {
+		await getPictures([{
+			author: {
+				picture: '/unknownshit.lolz'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir,
+			extension: "png"
+		})
+
+		const output = await getFiles(dumpdir)
+		expect(output.length).toBe(0)
+	})
+
+	test("getPictures adds extension if not yet present", async () => {
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir,
+			extension: "png"
+		})
+
+		const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg.png`).toString()
+		expect(output).not.toBe(null)
+	})
+
+	test("getPictures does not override existing picture if flag is set", async () => {
+		fs.mkdirSync(`${dumpdir}/img`)
+		fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir
+		})
+
+		const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
+		expect(output).toBe("dummy")
+	})
+
+	test("getPictures downloads author picture if present and overrides", async () => {
+		fs.mkdirSync(`${dumpdir}/img`)
+		fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
+
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}, {
+			author: {
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir
+		})
+
+		const output = await getFiles(dumpdir)
+		expect(output.length).toBe(1)
+
+		const firstFile = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
+		expect(output).not.toBe("dummy")
+	})
+
+})

yarn.lock

@@ -1329,6 +1329,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cacheable-request@npm:^7.0.2":
+  version: 7.0.2
+  resolution: "cacheable-request@npm:7.0.2"
+  dependencies:
+    clone-response: ^1.0.2
+    get-stream: ^5.1.0
+    http-cache-semantics: ^4.0.0
+    keyv: ^4.0.0
+    lowercase-keys: ^2.0.0
+    normalize-url: ^6.0.1
+    responselike: ^2.0.0
+  checksum: 176a1fceb987f1fee8b512ee7908445854a0c75854a11710f0d8de104cf840fd92e3c94ecd1f9144e57a25e17f5d72056591e5b33aabb8775061f906b0696a50
+  languageName: node
+  linkType: hard
+
 "callsites@npm:^3.0.0":
   version: 3.1.0
   resolution: "callsites@npm:3.1.0"
@@ -2501,7 +2516,26 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
-"got@npm:^11.8.2, got@npm:~11.8.2":
+"got@npm:11.8.3":
+  version: 11.8.3
+  resolution: "got@npm:11.8.3"
+  dependencies:
+    "@sindresorhus/is": ^4.0.0
+    "@szmarczak/http-timer": ^4.0.5
+    "@types/cacheable-request": ^6.0.1
+    "@types/responselike": ^1.0.0
+    cacheable-lookup: ^5.0.3
+    cacheable-request: ^7.0.2
+    decompress-response: ^6.0.0
+    http2-wrapper: ^1.0.0-beta.5.2
+    lowercase-keys: ^2.0.0
+    p-cancelable: ^2.0.0
+    responselike: ^2.0.0
+  checksum: 9c7e94bc0828f505de52451f445664394aee83c2051e033413a5fead1128814f322976de96905c3b2e74e6b91b122ab10f3c6b7806a15e49429e40440181c844
+  languageName: node
+  linkType: hard
+
+"got@npm:~11.8.2":
   version: 11.8.2
   resolution: "got@npm:11.8.2"
   dependencies:
@@ -3159,7 +3193,7 @@ fsevents@^2.1.2:
     ejs: ^3.1.6
     ent: ^2.2.0
     fast-xml-parser: ^3.18.0
-    got: ^11.8.2
+    got: 11.8.3
     howlongtobeat: ^1.3.1
     imagemagick: ^0.1.3
     jest: ^26.6.3
@@ -4265,6 +4299,13 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
+"normalize-url@npm:^6.0.1":
+  version: 6.1.0
+  resolution: "normalize-url@npm:6.1.0"
+  checksum: 5fb69e98c149f4a54a7bb0f1904cc524627c0d23327a9feafacacf135d01d9595c65e80ced6f27c17c1959541ea732815b604ff8a6ec52ec3fe7a391b92cfba9
+  languageName: node
+  linkType: hard
+
 "npm-run-path@npm:^2.0.0":
   version: 2.0.2
   resolution: "npm-run-path@npm:2.0.2"