add webmention getpicture functionality
This commit is contained in:
parent
69bf2bf61b
commit
66d0fd3e69
|
@ -9,3 +9,4 @@ dump
|
||||||
!.yarn/versions
|
!.yarn/versions
|
||||||
|
|
||||||
*.sublime-workspace
|
*.sublime-workspace
|
||||||
|
node_modules/
|
|
@ -1 +1,2 @@
|
||||||
yarnPath: ".yarn/releases/yarn-berry.cjs"
|
yarnPath: ".yarn/releases/yarn-berry.cjs"
|
||||||
|
nodeLinker: node-modules
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "jam-my-stack",
|
"name": "jam-my-stack",
|
||||||
"version": "1.0.22",
|
"version": "1.0.25",
|
||||||
"repository": {
|
"repository": {
|
||||||
"url": "https://github.com/wgroeneveld/jam-my-stack",
|
"url": "https://github.com/wgroeneveld/jam-my-stack",
|
||||||
"type": "git"
|
"type": "git"
|
||||||
|
@ -16,7 +16,7 @@
|
||||||
"ejs": "^3.1.6",
|
"ejs": "^3.1.6",
|
||||||
"ent": "^2.2.0",
|
"ent": "^2.2.0",
|
||||||
"fast-xml-parser": "^3.18.0",
|
"fast-xml-parser": "^3.18.0",
|
||||||
"got": "^11.8.2",
|
"got": "11.8.3",
|
||||||
"howlongtobeat": "^1.3.1",
|
"howlongtobeat": "^1.3.1",
|
||||||
"imagemagick": "^0.1.3",
|
"imagemagick": "^0.1.3",
|
||||||
"lunr": "^2.3.9",
|
"lunr": "^2.3.9",
|
||||||
|
|
|
@ -6,6 +6,7 @@ const { thumbify } = require('./youtube/thumbify.js')
|
||||||
|
|
||||||
const { getWebmentions } = require('./webmention/get.js')
|
const { getWebmentions } = require('./webmention/get.js')
|
||||||
const { sendWebmentions } = require('./webmention/send.js')
|
const { sendWebmentions } = require('./webmention/send.js')
|
||||||
|
const { getPictures } = require('./webmention/pictures.js')
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
mastodon: {
|
mastodon: {
|
||||||
|
@ -25,6 +26,7 @@ module.exports = {
|
||||||
},
|
},
|
||||||
webmention: {
|
webmention: {
|
||||||
getWebmentions: getWebmentions,
|
getWebmentions: getWebmentions,
|
||||||
send: sendWebmentions
|
send: sendWebmentions,
|
||||||
|
getPictures
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
const got = require("got");
|
||||||
|
const { createWriteStream } = require("fs");
|
||||||
|
const { pipeline } = require('stream/promises');
|
||||||
|
const { existsSync } = require('fs');
|
||||||
|
const fsp = require('fs').promises;
|
||||||
|
const { dirname } = require('path');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* config example: {
|
||||||
|
* endpoint: "https://bla.com",
|
||||||
|
* directory: "dir/to/save/to",
|
||||||
|
* override: true, // optional
|
||||||
|
* extension: "jpg" // optional
|
||||||
|
* }
|
||||||
|
**/
|
||||||
|
async function getPictures(webmentions, config) {
|
||||||
|
const pics = await Promise.all(
|
||||||
|
webmentions.filter(wm => wm.author && wm.author.picture).map(async wm => {
|
||||||
|
let fileName = `${config.directory}/${wm.author.picture}`
|
||||||
|
if(config.extension && !fileName.endsWith(config.extension)) {
|
||||||
|
fileName += `.${config.extension}`
|
||||||
|
}
|
||||||
|
|
||||||
|
if(config.override || !existsSync(fileName)) {
|
||||||
|
try {
|
||||||
|
await fsp.mkdir(dirname(fileName), { recursive: true }),
|
||||||
|
// retry API with streams is ridiculously complicated: https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md
|
||||||
|
await pipeline(
|
||||||
|
got.stream(`${config.endpoint}/${wm.author.picture}`),
|
||||||
|
createWriteStream(fileName)
|
||||||
|
)
|
||||||
|
} catch(err) {
|
||||||
|
console.log(` !! unable to download ${wm.author.picture}: ${err.message}`)
|
||||||
|
await fsp.unlink(fileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fileName
|
||||||
|
})
|
||||||
|
)
|
||||||
|
return pics
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
getPictures
|
||||||
|
}
|
|
@ -1,7 +1,5 @@
|
||||||
|
|
||||||
const got = require('got')
|
const got = require('got')
|
||||||
const fsp = require('fs').promises
|
|
||||||
|
|
||||||
const dayjs = require('dayjs')
|
const dayjs = require('dayjs')
|
||||||
|
|
||||||
async function sendWebmentions(domain, config) {
|
async function sendWebmentions(domain, config) {
|
||||||
|
|
|
@ -9,11 +9,14 @@ describe("webmention receive scenario test", () => {
|
||||||
|
|
||||||
// this tests against the real endpoint, meaning the token has to be correct.
|
// this tests against the real endpoint, meaning the token has to be correct.
|
||||||
test("getWebmentions fetches anything at all", async () => {
|
test("getWebmentions fetches anything at all", async () => {
|
||||||
const result = await getWebmentions(domain, {
|
try {
|
||||||
token: 'miauwkes',
|
const result = await getWebmentions(domain, {
|
||||||
endpoint: 'https://jam.brainbaking.com'
|
token: process.env.WEBMENTION_TOKEN,
|
||||||
})
|
endpoint: 'https://jam.brainbaking.com'
|
||||||
//expect(result.length).toBeGreaterThan(-1)
|
})
|
||||||
|
//expect(result.length).toBeGreaterThan(-1)
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
})
|
})
|
||||||
|
|
|
@ -0,0 +1,91 @@
|
||||||
|
const fs = require('fs');
|
||||||
|
const fsp = require('fs').promises;
|
||||||
|
const { rmdir } = require('./../utils')
|
||||||
|
const { getFiles } = require('../../src/file-utils');
|
||||||
|
|
||||||
|
const dumpdir = `${__dirname}/dump`
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
if(fs.existsSync(dumpdir)) {
|
||||||
|
rmdir(dumpdir)
|
||||||
|
}
|
||||||
|
fs.mkdirSync(dumpdir)
|
||||||
|
})
|
||||||
|
|
||||||
|
jest.disableAutomock()
|
||||||
|
jest.unmock('got')
|
||||||
|
|
||||||
|
describe("webmention downloading of author pictures", () => {
|
||||||
|
|
||||||
|
const { getPictures } = require('./../../src/webmention/pictures')
|
||||||
|
|
||||||
|
test("getPictures does not write a file if error occurs", async () => {
|
||||||
|
await getPictures([{
|
||||||
|
author: {
|
||||||
|
picture: '/unknownshit.lolz'
|
||||||
|
}
|
||||||
|
}], {
|
||||||
|
endpoint: 'https://brainbaking.com/',
|
||||||
|
directory: dumpdir,
|
||||||
|
extension: "png"
|
||||||
|
})
|
||||||
|
|
||||||
|
const output = await getFiles(dumpdir)
|
||||||
|
expect(output.length).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("getPictures adds extension if not yet present", async () => {
|
||||||
|
await getPictures([{
|
||||||
|
author: {
|
||||||
|
picture: '/img/avatar.jpg'
|
||||||
|
}
|
||||||
|
}], {
|
||||||
|
endpoint: 'https://brainbaking.com/',
|
||||||
|
directory: dumpdir,
|
||||||
|
extension: "png"
|
||||||
|
})
|
||||||
|
|
||||||
|
const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg.png`).toString()
|
||||||
|
expect(output).not.toBe(null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("getPictures does not override existing picture if flag is set", async () => {
|
||||||
|
fs.mkdirSync(`${dumpdir}/img`)
|
||||||
|
fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
|
||||||
|
await getPictures([{
|
||||||
|
author: {
|
||||||
|
picture: '/img/avatar.jpg'
|
||||||
|
}
|
||||||
|
}], {
|
||||||
|
endpoint: 'https://brainbaking.com/',
|
||||||
|
directory: dumpdir
|
||||||
|
})
|
||||||
|
|
||||||
|
const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
|
||||||
|
expect(output).toBe("dummy")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("getPictures downloads author picture if present and overrides", async () => {
|
||||||
|
fs.mkdirSync(`${dumpdir}/img`)
|
||||||
|
fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
|
||||||
|
|
||||||
|
await getPictures([{
|
||||||
|
author: {
|
||||||
|
picture: '/img/avatar.jpg'
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
author: {
|
||||||
|
}
|
||||||
|
}], {
|
||||||
|
endpoint: 'https://brainbaking.com/',
|
||||||
|
directory: dumpdir
|
||||||
|
})
|
||||||
|
|
||||||
|
const output = await getFiles(dumpdir)
|
||||||
|
expect(output.length).toBe(1)
|
||||||
|
|
||||||
|
const firstFile = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
|
||||||
|
expect(output).not.toBe("dummy")
|
||||||
|
})
|
||||||
|
|
||||||
|
})
|
45
yarn.lock
45
yarn.lock
|
@ -1329,6 +1329,21 @@ __metadata:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"cacheable-request@npm:^7.0.2":
|
||||||
|
version: 7.0.2
|
||||||
|
resolution: "cacheable-request@npm:7.0.2"
|
||||||
|
dependencies:
|
||||||
|
clone-response: ^1.0.2
|
||||||
|
get-stream: ^5.1.0
|
||||||
|
http-cache-semantics: ^4.0.0
|
||||||
|
keyv: ^4.0.0
|
||||||
|
lowercase-keys: ^2.0.0
|
||||||
|
normalize-url: ^6.0.1
|
||||||
|
responselike: ^2.0.0
|
||||||
|
checksum: 176a1fceb987f1fee8b512ee7908445854a0c75854a11710f0d8de104cf840fd92e3c94ecd1f9144e57a25e17f5d72056591e5b33aabb8775061f906b0696a50
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"callsites@npm:^3.0.0":
|
"callsites@npm:^3.0.0":
|
||||||
version: 3.1.0
|
version: 3.1.0
|
||||||
resolution: "callsites@npm:3.1.0"
|
resolution: "callsites@npm:3.1.0"
|
||||||
|
@ -2501,7 +2516,26 @@ fsevents@^2.1.2:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"got@npm:^11.8.2, got@npm:~11.8.2":
|
"got@npm:11.8.3":
|
||||||
|
version: 11.8.3
|
||||||
|
resolution: "got@npm:11.8.3"
|
||||||
|
dependencies:
|
||||||
|
"@sindresorhus/is": ^4.0.0
|
||||||
|
"@szmarczak/http-timer": ^4.0.5
|
||||||
|
"@types/cacheable-request": ^6.0.1
|
||||||
|
"@types/responselike": ^1.0.0
|
||||||
|
cacheable-lookup: ^5.0.3
|
||||||
|
cacheable-request: ^7.0.2
|
||||||
|
decompress-response: ^6.0.0
|
||||||
|
http2-wrapper: ^1.0.0-beta.5.2
|
||||||
|
lowercase-keys: ^2.0.0
|
||||||
|
p-cancelable: ^2.0.0
|
||||||
|
responselike: ^2.0.0
|
||||||
|
checksum: 9c7e94bc0828f505de52451f445664394aee83c2051e033413a5fead1128814f322976de96905c3b2e74e6b91b122ab10f3c6b7806a15e49429e40440181c844
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
|
"got@npm:~11.8.2":
|
||||||
version: 11.8.2
|
version: 11.8.2
|
||||||
resolution: "got@npm:11.8.2"
|
resolution: "got@npm:11.8.2"
|
||||||
dependencies:
|
dependencies:
|
||||||
|
@ -3159,7 +3193,7 @@ fsevents@^2.1.2:
|
||||||
ejs: ^3.1.6
|
ejs: ^3.1.6
|
||||||
ent: ^2.2.0
|
ent: ^2.2.0
|
||||||
fast-xml-parser: ^3.18.0
|
fast-xml-parser: ^3.18.0
|
||||||
got: ^11.8.2
|
got: 11.8.3
|
||||||
howlongtobeat: ^1.3.1
|
howlongtobeat: ^1.3.1
|
||||||
imagemagick: ^0.1.3
|
imagemagick: ^0.1.3
|
||||||
jest: ^26.6.3
|
jest: ^26.6.3
|
||||||
|
@ -4265,6 +4299,13 @@ fsevents@^2.1.2:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"normalize-url@npm:^6.0.1":
|
||||||
|
version: 6.1.0
|
||||||
|
resolution: "normalize-url@npm:6.1.0"
|
||||||
|
checksum: 5fb69e98c149f4a54a7bb0f1904cc524627c0d23327a9feafacacf135d01d9595c65e80ced6f27c17c1959541ea732815b604ff8a6ec52ec3fe7a391b92cfba9
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"npm-run-path@npm:^2.0.0":
|
"npm-run-path@npm:^2.0.0":
|
||||||
version: 2.0.2
|
version: 2.0.2
|
||||||
resolution: "npm-run-path@npm:2.0.2"
|
resolution: "npm-run-path@npm:2.0.2"
|
||||||
|
|
Loading…
Reference in New Issue