const got = require("got"); const { createWriteStream } = require("fs"); const { pipeline } = require('stream/promises'); const { existsSync } = require('fs'); const fsp = require('fs').promises; const { dirname } = require('path'); /** * config example: { * endpoint: "https://bla.com", * directory: "dir/to/save/to", * override: true, // optional * extension: "jpg" // optional * } **/ async function getPictures(webmentions, config) { const pics = await Promise.all( webmentions.filter(wm => wm.author && wm.author.picture).map(async wm => { let fileName = `${config.directory}/${wm.author.picture}` if(config.extension && !fileName.endsWith(config.extension)) { fileName += `.${config.extension}` } if(config.override || !existsSync(fileName)) { try { await fsp.mkdir(dirname(fileName), { recursive: true }), // retry API with streams is ridiculously complicated: https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md await pipeline( got.stream(`${config.endpoint}/${wm.author.picture}`), createWriteStream(fileName) ) } catch(err) { console.log(` !! unable to download ${wm.author.picture}: ${err.message}`) await fsp.unlink(fileName) } } return fileName }) ) return pics } module.exports = { getPictures }