From e577012002f10361ace883e6fc21f2100eca4f84 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Sat, 20 Mar 2021 13:11:22 +0100 Subject: [PATCH] add the option ignoreReplies --- README.md | 3 +- package.json | 2 +- src/mastodon/feed-parser.js | 4 +- src/webmention/send.js | 4 +- test/__mocks__/masto-feed-with-replies.xml | 112 +++++++++++++++++++++ test/mastodon/feed-parser.test.js | 23 +++++ 6 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 test/__mocks__/masto-feed-with-replies.xml diff --git a/README.md b/README.md index 34bc3f3..ddb3eff 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,12 @@ Usage example: }) ``` -Default values: +Options and their default values: - `utcOffset`: `60` (= GMT+1, that's where I am!) (in **minutes**, see [day.js docs](https://day.js.org/docs/en/manipulate/utc-offset) - `titleCount`: `50`. Will add "..." and trim if title length bigger. - `titlePrefix`: `""`. Will add before title (e.g. "Note: ") +- `ignoreReplies`: `false`. If true, will not process `in-reply-to` items. Note that this **does not** delete the notes dir with every call. It simply checks if there isn't already a file with the same name (based on the publication date), and adds one if not. diff --git a/package.json b/package.json index cafd0af..916e8f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jam-my-stack", - "version": "1.0.13", + "version": "1.0.14", "repository": { "url": "https://github.com/wgroeneveld/jam-my-stack", "type": "git" diff --git a/src/mastodon/feed-parser.js b/src/mastodon/feed-parser.js index 8e7264b..2960ee2 100644 --- a/src/mastodon/feed-parser.js +++ b/src/mastodon/feed-parser.js @@ -50,9 +50,10 @@ function trimIfNeeded(title, count, prefix) { // utcOffset = 60 (in minutes) // titleCount = 50 // titlePrefix = "Note: " +// ignoreReplies = false async function parseMastoFeed(options) { - const { notesdir, url, utcOffset = 60, titleCount = 50, titlePrefix = "" } = options + const { notesdir, url, utcOffset = 60, titleCount = 50, titlePrefix = "", ignoreReplies = false } = options const notesroot = await getFiles(notesdir) const notes = notesroot @@ -100,6 +101,7 @@ async function parseMastoFeed(options) { day } }) + .filter(itm => ignoreReplies ? itm.context .length > 0 : true) .filter(itm => !notes.includes(`${itm.year}/${itm.month}/${itm.hash}`)) .forEach(itm => convertAtomItemToMd(itm, notesdir)) } diff --git a/src/webmention/send.js b/src/webmention/send.js index f994abb..0251f63 100644 --- a/src/webmention/send.js +++ b/src/webmention/send.js @@ -20,6 +20,7 @@ async function getSince(configfile) { async function updateSince(configfile) { const since = new Date().toISOString() await fsp.writeFile(configfile, JSON.stringify({ since }, null, 2), 'utf-8') + return since } async function sendWebmentions(domain, configfile) { @@ -28,7 +29,8 @@ async function sendWebmentions(domain, configfile) { // this is an async call and will return 202 to say "started sending them out". const result = await got.put(url) - await updateSince(configfile) + const updatedSince = await updateSince(configfile) + return updatedSince } module.exports = { diff --git a/test/__mocks__/masto-feed-with-replies.xml b/test/__mocks__/masto-feed-with-replies.xml new file mode 100644 index 0000000..6bb7284 --- /dev/null +++ b/test/__mocks__/masto-feed-with-replies.xml @@ -0,0 +1,112 @@ + + + + https://chat.brainbaking.com/users/wouter/feed.atom + wouter's timeline + 2021-03-02T16:18:46 + https://chat.brainbaking.com/media/f39bcd85-5098-45e2-b395-e274b712d512/headshot_2020.jpg + + + + https://chat.brainbaking.com/users/wouter + http://activitystrea.ms/schema/1.0/person + https://chat.brainbaking.com/users/wouter + wouter + Wouter Groeneveld + Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠. + Level 35 Brain Baker. Loving the smell of freshly baked thoughts (and bread) in the morning 🍞. Sometimes convincing others to bake their brain (and bread) too 🧠. + wouter + + + + + + true + + + + + + + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + https://chat.brainbaking.com/objects/a51e13ce-d618-4602-84f7-f398126510ff + Enjoyed an afternoon of oldskool Diablo II on the Europebattle servers. We did a few Mephisto run... + Enjoyed an afternoon of oldskool Diablo II on the Europebattle servers. We did a few Mephisto runs, managed to hit Hell, and I re-converetd my druid into a windy one. Good times! + 2021-03-14T16:41:53.518661Z + 2021-03-14T16:41:53.518661Z + + https://chat.brainbaking.com/contexts/6c75527a-613c-47a4-9922-47bac2cb9ee5 + + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + https://chat.brainbaking.com/objects/2e58289c-f5f0-415c-b2e1-62c74662aa16 + @StampedingLonghorn I tried to chase him away, but you know how that turned out... 😼 There&#39;s ... + <span class="h-card"><a class="u-url mention" data-user="A4nwg4LYyh4WgrJOXg" href="https://social.linux.pizza/@StampedingLonghorn" rel="ugc">@<span>StampedingLonghorn</span></a></span> I tried to chase him away, but you know how that turned out... 😼 There&#39;s even cat hair inside the cases... (to be clear: also unintentional) + 2021-03-02T16:18:46.658056Z + 2021-03-02T16:18:46.658056Z + + tag:mastodon.social,2021-03-01:objectId=224637732:objectType=Conversation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/mastodon/feed-parser.test.js b/test/mastodon/feed-parser.test.js index 4ac8e4c..6902504 100644 --- a/test/mastodon/feed-parser.test.js +++ b/test/mastodon/feed-parser.test.js @@ -18,6 +18,29 @@ describe("mastodon feed parser tests", () => { fs.mkdirSync(dumpdir) }); + describe("ignore replies option", () => { + test("ignore replies if ignoreReplies is set to true", async () => { + await parseMastoFeed({ + url: "masto-feed-with-replies", + notesdir: dumpdir, + ignoreReplies: true + }) + + dir = await fsp.readdir(`${dumpdir}/2021/03`, { withFileTypes: true }) + expect(dir.length).toBe(1) + }) + test("does not ignore replies if ignoreReplies is set to false", async () => { + await parseMastoFeed({ + url: "masto-feed-with-replies", + notesdir: dumpdir, + ignoreReplies: false + }) + + dir = await fsp.readdir(`${dumpdir}/2021/03`, { withFileTypes: true }) + expect(dir.length).toBe(2) + }) + }) + test("parse embedded images", async () => { await parseMastoFeed({ url: "masto-feed-images",