howlongtobeat endpoint changed, up dep

.gitignore

@@ -9,3 +9,4 @@ dump
 !.yarn/versions
 
 *.sublime-workspace
+node_modules/

.yarnrc.yml

@@ -1 +1,2 @@
 yarnPath: ".yarn/releases/yarn-berry.cjs"
+nodeLinker: node-modules

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jam-my-stack",
-  "version": "1.0.21",
+  "version": "1.0.29",
   "repository": {
     "url": "https://github.com/wgroeneveld/jam-my-stack",
     "type": "git"
@@ -16,8 +16,8 @@
     "ejs": "^3.1.6",
     "ent": "^2.2.0",
     "fast-xml-parser": "^3.18.0",
-    "got": "^11.8.2",
-    "howlongtobeat": "^1.3.1",
+    "got": "11.8.3",
+    "howlongtobeat": "^1.5.1",
     "imagemagick": "^0.1.3",
     "lunr": "^2.3.9",
     "parser-front-matter": "^1.6.4",

src/index.js

@@ -6,6 +6,7 @@ const { thumbify } = require('./youtube/thumbify.js')
 
 const { getWebmentions } = require('./webmention/get.js')
 const { sendWebmentions } = require('./webmention/send.js')
+const { getPictures } = require('./webmention/pictures.js')
 
 module.exports = {
 	mastodon: {
@@ -25,6 +26,7 @@ module.exports = {
 	},
 	webmention: {
 		getWebmentions: getWebmentions,
-		send: sendWebmentions
+		send: sendWebmentions,
+		getPictures
 	}
 };

src/webmention/pictures.js

@@ -0,0 +1,47 @@
+const got = require("got");
+const { createWriteStream } = require("fs");
+const { pipeline } = require('stream/promises');
+const { existsSync } = require('fs');
+const fsp = require('fs').promises;
+const { dirname } = require('path');
+
+/**
+ * config example: {
+ * 		endpoint: "https://bla.com",
+ * 		directory: "dir/to/save/to",
+ * 		override: true, 	// optional
+ * 		extension: "jpg" 	// optional
+ * }
+ **/
+async function getPictures(webmentions, config) {
+	const pics = await Promise.all(
+		[...new Set(webmentions.filter(wm => wm.author && wm.author.picture).map(wm => wm.author.picture))]
+		.map(async picture => {
+			let fileName = `${config.directory}/${picture}`
+			if(config.extension && !fileName.endsWith(config.extension)) {
+				fileName += `.${config.extension}`
+			}
+
+			// TODO existsSync does not work, wrapped in Promise.all - does not wait for it!
+			if(config.override || !existsSync(fileName)) {
+				try {
+					await fsp.mkdir(dirname(fileName), { recursive: true }),
+					// retry API with streams is ridiculously complicated: https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md
+					await pipeline(
+						got.stream(`${config.endpoint}/${picture}`),
+						createWriteStream(fileName)
+					)
+				} catch(err) {
+					console.log(` !! unable to download ${picture}: ${err.message}`)
+					try { await fsp.unlink(fileName) } catch { }
+				}
+			}
+			return fileName
+		})
+	)
+	return pics
+}
+
+module.exports = {
+	getPictures
+}

src/webmention/send.js

@@ -1,7 +1,5 @@
 
 const got = require('got')
-const fsp = require('fs').promises
-
 const dayjs = require('dayjs')
 
 async function sendWebmentions(domain, config) {

test/webmention/get-e2e.test.js

@@ -9,11 +9,14 @@ describe("webmention receive scenario test", () => {
 
 	// this tests against the real endpoint, meaning the token has to be correct.
 	test("getWebmentions fetches anything at all", async () => {
-		const result = await getWebmentions(domain, {
-			token: 'miauwkes',
-			endpoint: 'https://jam.brainbaking.com'			
-		})
-		//expect(result.length).toBeGreaterThan(-1)
+		try {
+			const result = await getWebmentions(domain, {
+				token: process.env.WEBMENTION_TOKEN,
+				endpoint: 'https://jam.brainbaking.com'			
+			})
+			//expect(result.length).toBeGreaterThan(-1)
+		} catch {
+		}
 	})
 
 })

test/webmention/get.test.js

@@ -10,7 +10,7 @@ describe("webmention receive serve-my-jam tests", () => {
 
 	test("getWebmentions fetches from serve-my-jam depending on config", async () => {
 		const result = await getWebmentions(domain, {
-			token: 'miauwkes',
+			token: 'lol',
 			endpoint: 'https://jam.brainbaking.com'
 		})
 		expect(result.length).toBe(4)
@@ -18,7 +18,7 @@ describe("webmention receive serve-my-jam tests", () => {
 
 	test("getWebmentions enriches data with relatiave url", async () => {
 		const result = await getWebmentions(domain, {
-			token: 'miauwkes',
+			token: 'lol',
 			endpoint: 'https://jam.brainbaking.com/'			
 		})
 		const mention = result[0]
@@ -29,7 +29,7 @@ describe("webmention receive serve-my-jam tests", () => {
 
 	test("getWebmentions are sorted by published date descending", async() => {
 		const result = await getWebmentions(domain, {
-			token: 'miauwkes',
+			token: 'lol',
 			endpoint: 'https://jam.brainbaking.com'			
 		})
 

test/webmention/pictures.test.js

@@ -0,0 +1,91 @@
+const fs = require('fs');
+const fsp = require('fs').promises;
+const { rmdir } = require('./../utils')
+const { getFiles } = require('../../src/file-utils');
+
+const dumpdir = `${__dirname}/dump`
+
+beforeEach(async () => {
+	if(fs.existsSync(dumpdir)) {
+		rmdir(dumpdir)
+	}
+	fs.mkdirSync(dumpdir)
+})
+
+jest.disableAutomock()
+jest.unmock('got')
+
+describe("webmention downloading of author pictures", () => {
+
+	const { getPictures } = require('./../../src/webmention/pictures')
+
+	test("getPictures does not write a file if error occurs", async () => {
+		await getPictures([{
+			author: {
+				picture: '/unknownshit.lolz'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir,
+			extension: "png"
+		})
+
+		const output = await getFiles(dumpdir)
+		expect(output.length).toBe(0)
+	})
+
+	test("getPictures adds extension if not yet present", async () => {
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir,
+			extension: "png"
+		})
+
+		const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg.png`).toString()
+		expect(output).not.toBe(null)
+	})
+
+	test("getPictures does not override existing picture if flag is set", async () => {
+		fs.mkdirSync(`${dumpdir}/img`)
+		fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir
+		})
+
+		const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
+		expect(output).toBe("dummy")
+	})
+
+	test("getPictures downloads author picture if present and overrides", async () => {
+		fs.mkdirSync(`${dumpdir}/img`)
+		fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
+
+		await getPictures([{
+			author: {
+				picture: '/img/avatar.jpg'
+			}
+		}, {
+			author: {
+			}
+		}], {
+			endpoint: 'https://brainbaking.com/',
+			directory: dumpdir
+		})
+
+		const output = await getFiles(dumpdir)
+		expect(output.length).toBe(1)
+
+		const firstFile = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
+		expect(output).not.toBe("dummy")
+	})
+
+})

yarn.lock

@@ -1066,6 +1066,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"axios@npm:^0.22":
+  version: 0.22.0
+  resolution: "axios@npm:0.22.0"
+  dependencies:
+    follow-redirects: ^1.14.4
+  checksum: 8238c34fe3cec0ac3a09b00c1c6b355f99d37f9d3950dc570803ef4b4a6dd4971462ec94ecd2b3a89f4b13cda2568d8ac6d96abffb71ae2f0e9962bf5806b2e0
+  languageName: node
+  linkType: hard
+
 "babel-jest@npm:^26.6.3":
   version: 26.6.3
   resolution: "babel-jest@npm:26.6.3"
@@ -1329,6 +1338,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cacheable-request@npm:^7.0.2":
+  version: 7.0.2
+  resolution: "cacheable-request@npm:7.0.2"
+  dependencies:
+    clone-response: ^1.0.2
+    get-stream: ^5.1.0
+    http-cache-semantics: ^4.0.0
+    keyv: ^4.0.0
+    lowercase-keys: ^2.0.0
+    normalize-url: ^6.0.1
+    responselike: ^2.0.0
+  checksum: 176a1fceb987f1fee8b512ee7908445854a0c75854a11710f0d8de104cf840fd92e3c94ecd1f9144e57a25e17f5d72056591e5b33aabb8775061f906b0696a50
+  languageName: node
+  linkType: hard
+
 "callsites@npm:^3.0.0":
   version: 3.1.0
   resolution: "callsites@npm:3.1.0"
@@ -2327,6 +2351,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"follow-redirects@npm:^1.14.4":
+  version: 1.15.0
+  resolution: "follow-redirects@npm:1.15.0"
+  peerDependenciesMeta:
+    debug:
+      optional: true
+  checksum: 06e19a010288e3b0dcaf478a4ace2b4df20a5f3ca97c07026d63feda91121e0c0273ad3f293a6e5945ad3607b6d484a23061e79d6160b5406300e5f4357252d6
+  languageName: node
+  linkType: hard
+
 "for-in@npm:^1.0.2":
   version: 1.0.2
   resolution: "for-in@npm:1.0.2"
@@ -2501,7 +2535,26 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
-"got@npm:^11.8.2, got@npm:~11.8.2":
+"got@npm:11.8.3":
+  version: 11.8.3
+  resolution: "got@npm:11.8.3"
+  dependencies:
+    "@sindresorhus/is": ^4.0.0
+    "@szmarczak/http-timer": ^4.0.5
+    "@types/cacheable-request": ^6.0.1
+    "@types/responselike": ^1.0.0
+    cacheable-lookup: ^5.0.3
+    cacheable-request: ^7.0.2
+    decompress-response: ^6.0.0
+    http2-wrapper: ^1.0.0-beta.5.2
+    lowercase-keys: ^2.0.0
+    p-cancelable: ^2.0.0
+    responselike: ^2.0.0
+  checksum: 9c7e94bc0828f505de52451f445664394aee83c2051e033413a5fead1128814f322976de96905c3b2e74e6b91b122ab10f3c6b7806a15e49429e40440181c844
+  languageName: node
+  linkType: hard
+
+"got@npm:~11.8.2":
   version: 11.8.2
   resolution: "got@npm:11.8.2"
   dependencies:
@@ -2655,15 +2708,15 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
-"howlongtobeat@npm:^1.3.1":
-  version: 1.3.1
-  resolution: "howlongtobeat@npm:1.3.1"
+"howlongtobeat@npm:^1.5.1":
+  version: 1.5.1
+  resolution: "howlongtobeat@npm:1.5.1"
   dependencies:
+    axios: ^0.22
     cheerio: ^1.0.0-rc.2
     fast-levenshtein: ^2.0.6
-    request: ^2.88.0
     user-agents: ^1.0.580
-  checksum: ecd29eb8a45d44a508f803c94c0074b32473dbdc8bd36efebf608214d57df507dedad9a30967835d60dc359b83e7435697a3649e2c787610790e820d10cf3310
+  checksum: 19900d03e50365b0e9e5a682bab5cb9b03805fed9815acce21212ac20cffc9d24205b4063fa0864b38b0197c4517cf22bffd8495e0e5158bc01b50ae2884ee8d
   languageName: node
   linkType: hard
 
@@ -3159,8 +3212,8 @@ fsevents@^2.1.2:
     ejs: ^3.1.6
     ent: ^2.2.0
     fast-xml-parser: ^3.18.0
-    got: ^11.8.2
-    howlongtobeat: ^1.3.1
+    got: 11.8.3
+    howlongtobeat: ^1.5.1
     imagemagick: ^0.1.3
     jest: ^26.6.3
     lunr: ^2.3.9
@@ -4265,6 +4318,13 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
+"normalize-url@npm:^6.0.1":
+  version: 6.1.0
+  resolution: "normalize-url@npm:6.1.0"
+  checksum: 5fb69e98c149f4a54a7bb0f1904cc524627c0d23327a9feafacacf135d01d9595c65e80ced6f27c17c1959541ea732815b604ff8a6ec52ec3fe7a391b92cfba9
+  languageName: node
+  linkType: hard
+
 "npm-run-path@npm:^2.0.0":
   version: 2.0.2
   resolution: "npm-run-path@npm:2.0.2"
@@ -4786,7 +4846,7 @@ fsevents@^2.1.2:
   languageName: node
   linkType: hard
 
-"request@npm:^2.88.0, request@npm:^2.88.2":
+"request@npm:^2.88.2":
   version: 2.88.2
   resolution: "request@npm:2.88.2"
   dependencies: