add webmention getpicture functionality
parent
69bf2bf61b
commit
66d0fd3e69
|
@ -9,3 +9,4 @@ dump
|
|||
!.yarn/versions
|
||||
|
||||
*.sublime-workspace
|
||||
node_modules/
|
|
@ -1 +1,2 @@
|
|||
yarnPath: ".yarn/releases/yarn-berry.cjs"
|
||||
nodeLinker: node-modules
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "jam-my-stack",
|
||||
"version": "1.0.22",
|
||||
"version": "1.0.25",
|
||||
"repository": {
|
||||
"url": "https://github.com/wgroeneveld/jam-my-stack",
|
||||
"type": "git"
|
||||
|
@ -16,7 +16,7 @@
|
|||
"ejs": "^3.1.6",
|
||||
"ent": "^2.2.0",
|
||||
"fast-xml-parser": "^3.18.0",
|
||||
"got": "^11.8.2",
|
||||
"got": "11.8.3",
|
||||
"howlongtobeat": "^1.3.1",
|
||||
"imagemagick": "^0.1.3",
|
||||
"lunr": "^2.3.9",
|
||||
|
|
|
@ -6,6 +6,7 @@ const { thumbify } = require('./youtube/thumbify.js')
|
|||
|
||||
const { getWebmentions } = require('./webmention/get.js')
|
||||
const { sendWebmentions } = require('./webmention/send.js')
|
||||
const { getPictures } = require('./webmention/pictures.js')
|
||||
|
||||
module.exports = {
|
||||
mastodon: {
|
||||
|
@ -25,6 +26,7 @@ module.exports = {
|
|||
},
|
||||
webmention: {
|
||||
getWebmentions: getWebmentions,
|
||||
send: sendWebmentions
|
||||
send: sendWebmentions,
|
||||
getPictures
|
||||
}
|
||||
};
|
||||
|
|
|
@ -0,0 +1,45 @@
|
|||
const got = require("got");
|
||||
const { createWriteStream } = require("fs");
|
||||
const { pipeline } = require('stream/promises');
|
||||
const { existsSync } = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const { dirname } = require('path');
|
||||
|
||||
/**
|
||||
* config example: {
|
||||
* endpoint: "https://bla.com",
|
||||
* directory: "dir/to/save/to",
|
||||
* override: true, // optional
|
||||
* extension: "jpg" // optional
|
||||
* }
|
||||
**/
|
||||
async function getPictures(webmentions, config) {
|
||||
const pics = await Promise.all(
|
||||
webmentions.filter(wm => wm.author && wm.author.picture).map(async wm => {
|
||||
let fileName = `${config.directory}/${wm.author.picture}`
|
||||
if(config.extension && !fileName.endsWith(config.extension)) {
|
||||
fileName += `.${config.extension}`
|
||||
}
|
||||
|
||||
if(config.override || !existsSync(fileName)) {
|
||||
try {
|
||||
await fsp.mkdir(dirname(fileName), { recursive: true }),
|
||||
// retry API with streams is ridiculously complicated: https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md
|
||||
await pipeline(
|
||||
got.stream(`${config.endpoint}/${wm.author.picture}`),
|
||||
createWriteStream(fileName)
|
||||
)
|
||||
} catch(err) {
|
||||
console.log(` !! unable to download ${wm.author.picture}: ${err.message}`)
|
||||
await fsp.unlink(fileName)
|
||||
}
|
||||
}
|
||||
return fileName
|
||||
})
|
||||
)
|
||||
return pics
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getPictures
|
||||
}
|
|
@ -1,7 +1,5 @@
|
|||
|
||||
const got = require('got')
|
||||
const fsp = require('fs').promises
|
||||
|
||||
const dayjs = require('dayjs')
|
||||
|
||||
async function sendWebmentions(domain, config) {
|
||||
|
|
|
@ -9,11 +9,14 @@ describe("webmention receive scenario test", () => {
|
|||
|
||||
// this tests against the real endpoint, meaning the token has to be correct.
|
||||
test("getWebmentions fetches anything at all", async () => {
|
||||
const result = await getWebmentions(domain, {
|
||||
token: 'miauwkes',
|
||||
endpoint: 'https://jam.brainbaking.com'
|
||||
})
|
||||
//expect(result.length).toBeGreaterThan(-1)
|
||||
try {
|
||||
const result = await getWebmentions(domain, {
|
||||
token: process.env.WEBMENTION_TOKEN,
|
||||
endpoint: 'https://jam.brainbaking.com'
|
||||
})
|
||||
//expect(result.length).toBeGreaterThan(-1)
|
||||
} catch {
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const { rmdir } = require('./../utils')
|
||||
const { getFiles } = require('../../src/file-utils');
|
||||
|
||||
const dumpdir = `${__dirname}/dump`
|
||||
|
||||
beforeEach(async () => {
|
||||
if(fs.existsSync(dumpdir)) {
|
||||
rmdir(dumpdir)
|
||||
}
|
||||
fs.mkdirSync(dumpdir)
|
||||
})
|
||||
|
||||
jest.disableAutomock()
|
||||
jest.unmock('got')
|
||||
|
||||
describe("webmention downloading of author pictures", () => {
|
||||
|
||||
const { getPictures } = require('./../../src/webmention/pictures')
|
||||
|
||||
test("getPictures does not write a file if error occurs", async () => {
|
||||
await getPictures([{
|
||||
author: {
|
||||
picture: '/unknownshit.lolz'
|
||||
}
|
||||
}], {
|
||||
endpoint: 'https://brainbaking.com/',
|
||||
directory: dumpdir,
|
||||
extension: "png"
|
||||
})
|
||||
|
||||
const output = await getFiles(dumpdir)
|
||||
expect(output.length).toBe(0)
|
||||
})
|
||||
|
||||
test("getPictures adds extension if not yet present", async () => {
|
||||
await getPictures([{
|
||||
author: {
|
||||
picture: '/img/avatar.jpg'
|
||||
}
|
||||
}], {
|
||||
endpoint: 'https://brainbaking.com/',
|
||||
directory: dumpdir,
|
||||
extension: "png"
|
||||
})
|
||||
|
||||
const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg.png`).toString()
|
||||
expect(output).not.toBe(null)
|
||||
})
|
||||
|
||||
test("getPictures does not override existing picture if flag is set", async () => {
|
||||
fs.mkdirSync(`${dumpdir}/img`)
|
||||
fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
|
||||
await getPictures([{
|
||||
author: {
|
||||
picture: '/img/avatar.jpg'
|
||||
}
|
||||
}], {
|
||||
endpoint: 'https://brainbaking.com/',
|
||||
directory: dumpdir
|
||||
})
|
||||
|
||||
const output = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
|
||||
expect(output).toBe("dummy")
|
||||
})
|
||||
|
||||
test("getPictures downloads author picture if present and overrides", async () => {
|
||||
fs.mkdirSync(`${dumpdir}/img`)
|
||||
fs.writeFileSync(`${dumpdir}/img/avatar.jpg`, "dummy", "utf8")
|
||||
|
||||
await getPictures([{
|
||||
author: {
|
||||
picture: '/img/avatar.jpg'
|
||||
}
|
||||
}, {
|
||||
author: {
|
||||
}
|
||||
}], {
|
||||
endpoint: 'https://brainbaking.com/',
|
||||
directory: dumpdir
|
||||
})
|
||||
|
||||
const output = await getFiles(dumpdir)
|
||||
expect(output.length).toBe(1)
|
||||
|
||||
const firstFile = fs.readFileSync(`${dumpdir}/img/avatar.jpg`).toString()
|
||||
expect(output).not.toBe("dummy")
|
||||
})
|
||||
|
||||
})
|
45
yarn.lock
45
yarn.lock
|
@ -1329,6 +1329,21 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"cacheable-request@npm:^7.0.2":
|
||||
version: 7.0.2
|
||||
resolution: "cacheable-request@npm:7.0.2"
|
||||
dependencies:
|
||||
clone-response: ^1.0.2
|
||||
get-stream: ^5.1.0
|
||||
http-cache-semantics: ^4.0.0
|
||||
keyv: ^4.0.0
|
||||
lowercase-keys: ^2.0.0
|
||||
normalize-url: ^6.0.1
|
||||
responselike: ^2.0.0
|
||||
checksum: 176a1fceb987f1fee8b512ee7908445854a0c75854a11710f0d8de104cf840fd92e3c94ecd1f9144e57a25e17f5d72056591e5b33aabb8775061f906b0696a50
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"callsites@npm:^3.0.0":
|
||||
version: 3.1.0
|
||||
resolution: "callsites@npm:3.1.0"
|
||||
|
@ -2501,7 +2516,26 @@ fsevents@^2.1.2:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"got@npm:^11.8.2, got@npm:~11.8.2":
|
||||
"got@npm:11.8.3":
|
||||
version: 11.8.3
|
||||
resolution: "got@npm:11.8.3"
|
||||
dependencies:
|
||||
"@sindresorhus/is": ^4.0.0
|
||||
"@szmarczak/http-timer": ^4.0.5
|
||||
"@types/cacheable-request": ^6.0.1
|
||||
"@types/responselike": ^1.0.0
|
||||
cacheable-lookup: ^5.0.3
|
||||
cacheable-request: ^7.0.2
|
||||
decompress-response: ^6.0.0
|
||||
http2-wrapper: ^1.0.0-beta.5.2
|
||||
lowercase-keys: ^2.0.0
|
||||
p-cancelable: ^2.0.0
|
||||
responselike: ^2.0.0
|
||||
checksum: 9c7e94bc0828f505de52451f445664394aee83c2051e033413a5fead1128814f322976de96905c3b2e74e6b91b122ab10f3c6b7806a15e49429e40440181c844
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"got@npm:~11.8.2":
|
||||
version: 11.8.2
|
||||
resolution: "got@npm:11.8.2"
|
||||
dependencies:
|
||||
|
@ -3159,7 +3193,7 @@ fsevents@^2.1.2:
|
|||
ejs: ^3.1.6
|
||||
ent: ^2.2.0
|
||||
fast-xml-parser: ^3.18.0
|
||||
got: ^11.8.2
|
||||
got: 11.8.3
|
||||
howlongtobeat: ^1.3.1
|
||||
imagemagick: ^0.1.3
|
||||
jest: ^26.6.3
|
||||
|
@ -4265,6 +4299,13 @@ fsevents@^2.1.2:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"normalize-url@npm:^6.0.1":
|
||||
version: 6.1.0
|
||||
resolution: "normalize-url@npm:6.1.0"
|
||||
checksum: 5fb69e98c149f4a54a7bb0f1904cc524627c0d23327a9feafacacf135d01d9595c65e80ced6f27c17c1959541ea732815b604ff8a6ec52ec3fe7a391b92cfba9
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"npm-run-path@npm:^2.0.0":
|
||||
version: 2.0.2
|
||||
resolution: "npm-run-path@npm:2.0.2"
|
||||
|
|
Loading…
Reference in New Issue