From 62e49c5c1514cb6150150e282ecbf37cd0252e85 Mon Sep 17 00:00:00 2001
From: wgroeneveld <wouter@brainbaking.com>
Date: Wed, 24 Mar 2021 15:34:08 +0100
Subject: [PATCH] integrate pingback sending into PUT /webmention

---
 README.md                                     | 43 ++++++++++++++++--
 src/linkdiscoverer.js                         | 31 +++++++++----
 src/pingback/send.js                          | 29 ++++++++++--
 src/webmention/send.js                        | 34 ++++++++------
 test/__mocks__/index.xml                      |  2 +
 test/__mocks__/link-discover-bothtypes.html   | 12 +++++
 .../pingback-discover-test-headers.json       |  3 ++
 .../pingback-discover-test-multiple.html      | 12 +++++
 .../pingback-discover-test-single.html        | 11 +++++
 test/__mocks__/pingback-discover-test.html    | 12 +++++
 test/linkdiscoverer.test.js                   | 44 ++++++++++++++++---
 test/webmention/send.test.js                  | 23 +++++++++-
 12 files changed, 219 insertions(+), 37 deletions(-)
 create mode 100644 test/__mocks__/link-discover-bothtypes.html
 create mode 100644 test/__mocks__/pingback-discover-test-headers.json
 create mode 100644 test/__mocks__/pingback-discover-test-multiple.html
 create mode 100644 test/__mocks__/pingback-discover-test-single.html
 create mode 100644 test/__mocks__/pingback-discover-test.html

diff --git a/README.md b/README.md
index c8ec18f..8c76335 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ If you want to support the older pingback protocol, you can leverage webmenton.i
 
 #### 1.1 `POST /webmention`
 
-Post a webmention. Includes a _lot_ of cross-checking and validating to guard against possible spam. See the [W3C WebMention spec](https://www.w3.org/TR/webmention/#sender-notifies-receiver) - or the source - for details.
+Receive a webmention. Includes a _lot_ of cross-checking and validating to guard against possible spam. See the [W3C WebMention spec](https://www.w3.org/TR/webmention/#sender-notifies-receiver) - or the source - for details.
 
 Accepted form format: 
 
@@ -42,14 +42,14 @@ Retrieves a JSON array with relevant webmentions stored for that domain. The tok
 
 #### 1.3 `PUT /webmention/:domain/:token`
 
-Sends out webmentions, based on the domain's `index.xml` RSS feed, and optionally, a `since` request query parameter that is supposed to be a string, fed through [Dayjs](https://day.js.org/) to format. (e.g. `2021-03-16T16:00:00.000Z`). 
+Sends out **both webmentions and pingbacks**, based on the domain's `index.xml` RSS feed, and optionally, a `since` request query parameter that is supposed to be a string, fed through [Dayjs](https://day.js.org/) to format. (e.g. `2021-03-16T16:00:00.000Z`). 
 
 This does a couple of things:
 
 1. Fetch RSS entries (since x, or everything)
 2. Find outbound `href`s (starting with `http`)
-3. Check if those domains have a `webmention` link endpoint installed, according to the w3.org rules. 
-4. If so: `POST` for each found href with `source` the own domain and `target` the outbound link found in the RSS feed. 
+3. Check if those domains have a `webmention` link endpoint installed, according to the w3.org rules. If not, check for a `pingback` endpoint. If not, bail out.
+4. If webmention/pingback found: `POST` for each found href with `source` the own domain and `target` the outbound link found in the RSS feed, using either XML or form data according to the protocol. 
 
 As with the `POST` call, will result in a `202 Accepted` and handles things async/in parallel. 
 
@@ -57,6 +57,41 @@ As with the `POST` call, will result in a `202 Accepted` and handles things asyn
 
 Yes and no. It checks the `<pubDate/>` `<item/>` RSS tag by default, but if a `<time datetime="..."/>` tag is present in the `<description/>`, it treats that date as the "last modified" date. There is no such thing in the RSS 2.0 W3.org specs, so I had to come up with my own hacks! Remember that if you want this to work, you also need to include a time tag in your RSS feed (e.g. `.Lastmod` gitinfo in Hugo). 
 
+### 2. Pingbacks
+
+Pingbacks are in here for two reasons:
+
+1. I wanted to see how difficult it was to implement them. Turns out to be almost exactly the same as webmentions. This means the "new" W3 standards for webmentions are just as crappy as pingbacks... What's the difference between a form POST and an XML POST? Form factor?
+2. Much more blogs (Wordpress-alike) support only pingbacks. 
+
+#### 2.1 `POST /pingback`
+
+Receive a pingback. Includes a _lot_ of cross-checking and validating to guard against possible spam. Internally, converts it into a webmention and processes it just like that.
+
+Accepted XML body: 
+
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<methodCall>
+    <methodName>pingback.ping</methodName>
+    <params>
+        <param>
+            <value><string>https://brainbaking.com/kristien.html</string></value>
+        </param>
+        <param>
+            <value><string>https://kristienthoelen.be/2021/03/22/de-stadia-van-een-burn-out-in-welk-stadium-zit-jij/</string></value>
+        </param>
+    </params>
+</methodCall>
+```
+
+Will result in a `200 OK` - that returns XML according to [The W3 pingback XML-RPC spec](https://www.hixie.ch/specs/pingback/pingback#refsXMLRPC). Processes async. 
+
+#### 2.2 Sending pingbacks
+
+Happens automatically through `PUT /webmention/:domain/:token`! Links that are discovered as `rel="pingback"` that **do not** already have a webmention link will be processed as XML-RPC requests to be send. 
+
+
 ## TODOs
 
 - `published` date is not well-formatted and blindly taken over from feed
diff --git a/src/linkdiscoverer.js b/src/linkdiscoverer.js
index 18ed621..e3b2917 100644
--- a/src/linkdiscoverer.js
+++ b/src/linkdiscoverer.js
@@ -9,30 +9,43 @@ const baseUrlOf = (url) => {
 	return split[0] + '//' + split[2]
 }
 
+const buildWebmentionHeaderLink = (link) => {
+	// e.g. Link: <http://aaronpk.example/webmention-endpoint>; rel="webmention"
+	return link
+		.split(";")[0]
+		.replace("<" ,"")
+		.replace(">", "")
+}
+
 // see https://www.w3.org/TR/webmention/#sender-discovers-receiver-webmention-endpoint
 async function discover(target) {
 	try {
 		const endpoint = await got(target)
 		if(endpoint.headers.link?.indexOf("webmention") >= 0) {
-			// e.g. Link: <http://aaronpk.example/webmention-endpoint>; rel="webmention"
-			const link = endpoint.headers.link
-				.split(";")[0]
-				.replace("<" ,"")
-				.replace(">", "")
 			return {
-				link,
+				link: buildWebmentionHeaderLink(endpoint.headers.link),
 				type: "webmention"
 			}
+		} else if(endpoint.headers["X-Pingback"]) {
+			return {
+				link: endpoint.headers["X-Pingback"],
+				type: "pingback"
+			}
 		}
 
 		const format = mf2(endpoint.body, {
 			// this also complies with w3.org regulations: relative endpoint could be possible
 			baseUrl: baseUrlOf(target)
 		})
-		const link = format.rels?.webmention?.[0]
+		const webmention = format.rels?.webmention?.[0]
+		const pingback = format.rels?.pingback?.[0]
+		if(!webmention && !pingback) {
+			throw "no webmention and no pingback found?"
+		}
+
 		return {
-			link,
-			type: "webmention"
+			link: webmention ? webmention : pingback,
+			type: webmention ? "webmention" : "pingback"
 		}
 	} catch(err) {
 		console.warn(` -- whoops, failed to discover ${target}, why: ${err}`)
diff --git a/src/pingback/send.js b/src/pingback/send.js
index ba664b4..342e73e 100644
--- a/src/pingback/send.js
+++ b/src/pingback/send.js
@@ -1,9 +1,30 @@
 
-async function send(domain, since) {
-	const feed = await got(`https://${domain}/index.xml`)
-	await parseRssFeed(feed.body, since)
+const got = require('got')
+
+async function sendPingbackToEndpoint(endpoint, source, target) {
+	const body = `<?xml version="1.0" encoding="UTF-8"?>
+<methodCall>
+	<methodName>pingback.ping</methodName>
+	<params>
+		<param>
+			<value><string>${source}</string></value>
+		</param>
+		<param>
+			<value><string>${target}</string></value>
+		</param>
+	</params>
+</methodCall>`
+	await got.post(endpoint, {
+		contentType: "text/xml",
+		body,
+		retry: {
+			limit: 5,
+			methods: ["POST"]
+		}
+	})
+	console.log(` OK: pingback@${endpoint}, sent: source ${source}, target ${target}`)
 }
 
 module.exports = {
-	send
+	sendPingbackToEndpoint
 }
diff --git a/src/webmention/send.js b/src/webmention/send.js
index 3c69e02..8de5c97 100644
--- a/src/webmention/send.js
+++ b/src/webmention/send.js
@@ -2,24 +2,32 @@
 const got = require('got')
 const { collect } = require('./rsslinkcollector')
 const { discover } = require('./../linkdiscoverer')
+const { sendPingbackToEndpoint } = require('./../pingback/send')
+
+async function sendWebmentionToEndpoint(endpoint, source, target) {
+	await got.post(endpoint, {
+		contentType: "x-www-form-urlencoded",
+		form: {
+			source,
+			target
+		},
+		retry: {
+			limit: 5,
+			methods: ["POST"]
+		}
+	})
+	console.log(` OK: webmention@${endpoint}, sent: source ${source}, target ${target}`)
+}
 
 async function mention(opts) {
 	const { source, target } = opts
 	const endpoint = await discover(target)
-	if(endpoint) {
-		await got.post(endpoint.link, {
-			contentType: "x-www-form-urlencoded",
-			form: {
-				source,
-				target
-			},
-			retry: {
-				limit: 5,
-				methods: ["POST"]
-			}
-		})
-		console.log(` OK: ${endpoint.type}@${endpoint.link}, sent: source ${source}, target ${target}`)
+	if(!endpoint) return
+	const sendMention = {
+		"webmention": sendWebmentionToEndpoint,
+		"pingback": sendPingbackToEndpoint
 	}
+	await sendMention[endpoint.type](endpoint.link, source, target)
 }
 
 async function parseRssFeed(xml, since) {
diff --git a/test/__mocks__/index.xml b/test/__mocks__/index.xml
index e449348..e4e2f44 100644
--- a/test/__mocks__/index.xml
+++ b/test/__mocks__/index.xml
@@ -28,6 +28,8 @@
           
           <p>hi there! test discovering: <a href="https://brainbaking.com/link-discover-test-single.html">single</a>. Nice!</p>
 
+          <p>this one is a pingback-only <a href="https://brainbaking.com/pingback-discover-test-single.html">single</a> one. Not good!</p>
+
           <p>another cool link: <a href="https://brainbaking.com/link-discover-test-multiple.html">multiple</a></p>
           
           ]]>
diff --git a/test/__mocks__/link-discover-bothtypes.html b/test/__mocks__/link-discover-bothtypes.html
new file mode 100644
index 0000000..f56b990
--- /dev/null
+++ b/test/__mocks__/link-discover-bothtypes.html
@@ -0,0 +1,12 @@
+<html>
+<head>
+...
+<link href="http://aaronpk.example/webmention-endpoint" rel="webmention" />
+<link href="http://aaronpk.example/pingback-endpoint" rel="pingback" />
+...
+</head>
+<body>
+....
+...
+</body>
+</html>
diff --git a/test/__mocks__/pingback-discover-test-headers.json b/test/__mocks__/pingback-discover-test-headers.json
new file mode 100644
index 0000000..ac1d99f
--- /dev/null
+++ b/test/__mocks__/pingback-discover-test-headers.json
@@ -0,0 +1,3 @@
+{
+	"X-Pingback": "http://aaronpk.example/pingback-endpoint"
+}
\ No newline at end of file
diff --git a/test/__mocks__/pingback-discover-test-multiple.html b/test/__mocks__/pingback-discover-test-multiple.html
new file mode 100644
index 0000000..e8b6f85
--- /dev/null
+++ b/test/__mocks__/pingback-discover-test-multiple.html
@@ -0,0 +1,12 @@
+<html>
+<head>
+...
+<link href="http://aaronpk.example/pingback-endpoint-header" rel="pingback" />
+...
+</head>
+<body>
+....
+<a href="http://aaronpk.example/pingback-endpoint-body" rel="pingback">pingback</a>
+...
+</body>
+</html>
diff --git a/test/__mocks__/pingback-discover-test-single.html b/test/__mocks__/pingback-discover-test-single.html
new file mode 100644
index 0000000..b82f340
--- /dev/null
+++ b/test/__mocks__/pingback-discover-test-single.html
@@ -0,0 +1,11 @@
+<html>
+<head>
+...
+...
+</head>
+<body>
+....
+<a href="http://aaronpk.example/pingback-endpoint-body" rel="pingback">pingback</a>
+...
+</body>
+</html>
diff --git a/test/__mocks__/pingback-discover-test.html b/test/__mocks__/pingback-discover-test.html
new file mode 100644
index 0000000..e8b6f85
--- /dev/null
+++ b/test/__mocks__/pingback-discover-test.html
@@ -0,0 +1,12 @@
+<html>
+<head>
+...
+<link href="http://aaronpk.example/pingback-endpoint-header" rel="pingback" />
+...
+</head>
+<body>
+....
+<a href="http://aaronpk.example/pingback-endpoint-body" rel="pingback">pingback</a>
+...
+</body>
+</html>
diff --git a/test/linkdiscoverer.test.js b/test/linkdiscoverer.test.js
index 9eb47f5..4a3bb6d 100644
--- a/test/linkdiscoverer.test.js
+++ b/test/linkdiscoverer.test.js
@@ -3,6 +3,45 @@ const { discover } = require('../src/linkdiscoverer')
 
 describe("link discoverer", () => {
 
+	test("discover nothing if no link is present", async() => {
+		const result = await discover("https://brainbaking.com/link-discover-test-none.html")
+		expect(result).toBeUndefined()
+	})
+
+	test("prefer webmentions over pingbacks if both links are present", async () => {
+		const result = await discover("https://brainbaking.com/link-discover-bothtypes.html")
+		expect(result).toEqual({
+			link: "http://aaronpk.example/webmention-endpoint",
+			type: "webmention"
+		})
+	})
+
+	describe("discovers pingback links", () => {
+		test("discover link if present in header", async () => {
+			const result = await discover("https://brainbaking.com/pingback-discover-test.html")
+			expect(result).toEqual({
+				link: "http://aaronpk.example/pingback-endpoint",
+				type: "pingback"
+			})
+		})
+
+		test("discover link if sole entry somewhere in html", async () => {
+			const result = await discover("https://brainbaking.com/pingback-discover-test-single.html")
+			expect(result).toEqual({
+				link: "http://aaronpk.example/pingback-endpoint-body",
+				type: "pingback"
+			})
+		})
+
+		test("use link in header if multiple present in html", async () => {
+			const result = await discover("https://brainbaking.com/pingback-discover-test-multiple.html")
+			expect(result).toEqual({
+				link: "http://aaronpk.example/pingback-endpoint-header",
+				type: "pingback"
+			})
+		})				
+	})
+
 	describe("discovers webmention links", () => {
 		test("discover link if present in header", async () => {
 			const result = await discover("https://brainbaking.com/link-discover-test.html")
@@ -12,11 +51,6 @@ describe("link discoverer", () => {
 			})
 		})
 
-		test("discover nothing if no webmention link is present", async() => {
-			const result = await discover("https://brainbaking.com/link-discover-test-none.html")
-			expect(result).toBeUndefined()
-		})
-
 		test("discover link if sole entry somewhere in html", async () => {
 			const result = await discover("https://brainbaking.com/link-discover-test-single.html")
 			expect(result).toEqual({
diff --git a/test/webmention/send.test.js b/test/webmention/send.test.js
index 663ad71..241a8d0 100644
--- a/test/webmention/send.test.js
+++ b/test/webmention/send.test.js
@@ -5,13 +5,13 @@ const { send } = require('../../src/webmention/send')
 
 
 describe("webmention send scenarios", () => {
-	test("webmention send integration test", async () => {
+	test("webmention send integration test that can send both webmentions and pingbacks", async () => {
 		got.post = jest.fn()
 
 		// fetches index.xml
 		await send("brainbaking.com", '2021-03-16T16:00:00.000Z')
 
-		expect(got.post).toHaveBeenCalledTimes(2)
+		expect(got.post).toHaveBeenCalledTimes(3)
 		expect(got.post).toHaveBeenCalledWith("http://aaronpk.example/webmention-endpoint-header", {
 			contentType: "x-www-form-urlencoded",
 			form: {
@@ -23,6 +23,25 @@ describe("webmention send scenarios", () => {
 				methods: ["POST"]
 			}
 		})
+		expect(got.post).toHaveBeenCalledWith("http://aaronpk.example/pingback-endpoint-body", {
+			contentType: "text/xml",
+			body: `<?xml version="1.0" encoding="UTF-8"?>
+<methodCall>
+	<methodName>pingback.ping</methodName>
+	<params>
+		<param>
+			<value><string>https://brainbaking.com/notes/2021/03/16h17m07s14/</string></value>
+		</param>
+		<param>
+			<value><string>https://brainbaking.com/pingback-discover-test-single.html</string></value>
+		</param>
+	</params>
+</methodCall>`,
+			retry: {
+				limit: 5,
+				methods: ["POST"]
+			}
+		})
 		expect(got.post).toHaveBeenCalledWith("http://aaronpk.example/webmention-endpoint-body", {
 			contentType: "x-www-form-urlencoded",
 			form: {