const { getFiles, createIfNotExists } = require('./../file-utils'); const fs = require('fs').promises; const got = require("got"); const {promisify} = require('util'); const frontMatterParser = require('parser-front-matter'); const parse = promisify(frontMatterParser.parse.bind(frontMatterParser)); const exec = promisify(require('child_process').exec); const stream = require('stream'); const pipeline = promisify(stream.pipeline); const { createWriteStream } = require("fs"); async function loadPostsWithFrontMatter(postsDirectoryPath) { const postNames = await getFiles(postsDirectoryPath); const posts = await Promise.all( // could be .DS_Store stuff found using recursive function above... postNames.filter(name => name.endsWith('.md')).map(async fileName => { const fileContent = await fs.readFile(fileName, 'utf8'); const {content, data} = await parse(fileContent); return { game: data.game_name, howlongtobeat_id: data.howlongtobeat_id, file: fileName } }) ); return posts; } async function downloadThumbnail(url, id, dir) { if(url.indexOf('howlongtobeat.com') == -1) { url = `https://howlongtobeat.com/games/${url}` } console.log(` --- downloading thumbnail ${url} of id ${id}...`) await pipeline( got.stream(url), createWriteStream(`${dir}/cover.jpg`) ) } // I'm doing this myself, getting tired of HLTB's API breakages. // See https://github.com/ckatzorke/howlongtobeat/issues/40 async function howLongToBeatSearch(game) { const params = { "searchType": "games", "searchTerms": game.split(" "), "searchPage": 1, "size": 20, "searchOptions": { "games": { "userId": 0, "platform": "", "sortCategory": "popular", "rangeCategory": "main", "rangeTime": { "min": 0, "max": 0 }, "gameplay": { "perspective": "", "flow": "", "genre": "" }, "modifier": "" }, "users": { "sortCategory": "postcount" }, "filter": "", "sort": 0, "randomizer": 0 } } const results = await got.post("https://www.howlongtobeat.com/api/search", { json: params, headers: { 'Content-Type': "application/json", 'Referer': 'https://howlongtobeat.com/', "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:90.0) Gecko/20100101 Firefox/90.0" } }).json() return results.data.map(item => { return { 'id': item.game_id, 'name': item.game_name, 'imageUrl': item.game_image, 'howlong': (item.comp_main / 60 / 60).toFixed(1) } }) } async function fillInHowLongToBeat(posts, postRootDir, downloadRootDir) { for(post of posts) { const results = await howLongToBeatSearch(post.game) if(results.length > 0) { const game = results[0] post.howlongtobeat = game.howlong post.howlongtobeat_id = game.id if(downloadRootDir) { // this assumes postDir is something like ".../content/blah" and dlDir like ".../static/blah" to mirror its structure const downloadDir = post.file.replace(postRootDir, downloadRootDir).replace('.md', '') createIfNotExists(downloadDir) await downloadThumbnail(game.imageUrl, game.id, downloadDir) const { stdout, stderr } = await exec(`mogrify -sampling-factor 4:2:0 -strip -quality 80 -interlace JPEG -resize '>300x' -format jpg -colorspace sRGB ${downloadDir}/cover.jpg`) if(stderr) { console.log(`-- WARN: unable to mogrify downloaded JPG: ${stderr}`) } } } } } async function run(options) { const { postDir, downloadDir } = options console.log(`-- SCANNING posts in ${postDir} for "game_name" key... --`) let posts = await loadPostsWithFrontMatter(postDir) console.log(` >> Found ${posts.length}`) posts = posts.filter(post => post.game && !post.howlongtobeat_id) console.log(` >> ToProcess: ${posts.length}`) await fillInHowLongToBeat(posts, postDir, downloadDir) for(post of posts) { let data = await fs.readFile(post.file, 'utf8') // just in case it's there, do not duplicate keys! data = data.replace(/howlongtobeat_id:(.*)\n/, '') data = data.replace(/nhowlongtobeat_hrs:(.*)\n/, '') data = data.replace(/game_name:/, `howlongtobeat_id: ${post.howlongtobeat_id}\nhowlongtobeat_hrs: ${post.howlongtobeat}\ngame_name:`) console.log(`\tFound game ${post.game}, how long filling in: ${post.howlongtobeat} (id #${post.howlongtobeat_id})`) await fs.writeFile(post.file, data, 'utf8') } console.log("-- DONE modifying files --") } module.exports = { howlong: run }