From 95abdcc7f0f0a26c13f7716d251abe9ecc750073 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Wed, 7 Apr 2021 17:31:23 +0200 Subject: [PATCH] added logging, refactored in webmention struct, use async after validation --- .github/workflows/main.yml | 2 +- app/logging.go | 31 +++++++++++++++++++++++++ app/routes.go | 2 ++ app/server.go | 9 +++---- app/webmention/handler.go | 18 ++++++++++++-- app/webmention/receive.go | 40 ++++++++++++++++++++++++++++++++ app/webmention/validate.go | 13 +++++++++++ common/httputils.go | 37 +++++++++++++++++++++++++++++ go.mod | 5 +++- go.sum | 27 +++++++++++++++++++++ jsfork/src/webmention/receive.js | 32 +------------------------ main.go | 10 ++++++++ playground.go | 24 +++++++++++++++++++ 13 files changed, 211 insertions(+), 39 deletions(-) create mode 100644 app/logging.go create mode 100644 app/webmention/receive.go create mode 100644 common/httputils.go create mode 100644 playground.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c732c82..2bfe92c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,4 +8,4 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '^1.16.0' - - run: go test ./... + - run: go test ./... -v diff --git a/app/logging.go b/app/logging.go new file mode 100644 index 0000000..6449173 --- /dev/null +++ b/app/logging.go @@ -0,0 +1,31 @@ + +package app + +import ( + "net/http" + + "github.com/rs/zerolog/log" +) + +type loggingResponseWriter struct { + http.ResponseWriter + statusCode int +} + +// mimic ResponseWriter's WriteHeader to capture the code +func (lrw *loggingResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + lrw.ResponseWriter.WriteHeader(code) +} + +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + logWriter := &loggingResponseWriter{w, http.StatusOK} + next.ServeHTTP(logWriter, r) + log.Info(). + Str("url", r.RequestURI). + Str("method", r.Method). + Int("status", logWriter.statusCode). + Msg("handled") + }) +} diff --git a/app/routes.go b/app/routes.go index 0e19fe5..3084d5f 100644 --- a/app/routes.go +++ b/app/routes.go @@ -9,6 +9,7 @@ import ( // stole ideas from https://pace.dev/blog/2018/05/09/how-I-write-http-services-after-eight-years.html // not that contempt with passing conf, but can't create receivers on non-local types, and won't move specifics into package app +// https://blog.questionable.services/article/http-handler-error-handling-revisited/ is the better idea, but more work func (s *server) routes() { s.router.HandleFunc("/", index.Handle(s.conf)).Methods("GET") s.router.HandleFunc("/pingback", pingback.Handle(s.conf)).Methods("POST") @@ -16,3 +17,4 @@ func (s *server) routes() { s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandleGet(s.conf))).Methods("GET") s.router.HandleFunc("/webmention/{domain}/{token}", s.authorizedOnly(webmention.HandlePut(s.conf))).Methods("PUT") } + diff --git a/app/server.go b/app/server.go index 68e3124..6e1c28b 100644 --- a/app/server.go +++ b/app/server.go @@ -2,13 +2,13 @@ package app import ( - "fmt" "strconv" "net/http" - "github.com/gorilla/mux" - "github.com/wgroeneveld/go-jamming/common" + + "github.com/gorilla/mux" + "github.com/rs/zerolog/log" ) type server struct { @@ -36,7 +36,8 @@ func Start() { server.routes() http.Handle("/", r) + r.Use(loggingMiddleware) - fmt.Printf("Serving at port %d...\n", server.conf.Port) + log.Info().Int("port", server.conf.Port).Msg("Serving...") http.ListenAndServe(":" + strconv.Itoa(server.conf.Port), nil) } diff --git a/app/webmention/handler.go b/app/webmention/handler.go index ad59fba..ec30a81 100644 --- a/app/webmention/handler.go +++ b/app/webmention/handler.go @@ -24,9 +24,23 @@ func HandlePost(conf *common.Config) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { r.ParseForm() if !validate(r, r.Header, conf) { - http.Error(w, "400 bad request", http.StatusBadRequest) + common.BadRequest(w) + return } - fmt.Printf("%+v\n", r.Header) + + target := r.FormValue("target") + if !isValidTargetUrl(target) { + common.BadRequest(w) + return + } + + wm := &webmention{ + source: r.FormValue("source"), + target: target, + } + + go wm.receive() + common.Accept(w) } } diff --git a/app/webmention/receive.go b/app/webmention/receive.go new file mode 100644 index 0000000..c4e076a --- /dev/null +++ b/app/webmention/receive.go @@ -0,0 +1,40 @@ + +package webmention + +import ( + "fmt" + + "github.com/wgroeneveld/go-jamming/common" + + "github.com/rs/zerolog/log" +) + +type webmention struct { + source string + target string +} + +func (wm *webmention) String() string { + return fmt.Sprintf("source: %s, target: %s", wm.source, wm.target) +} + +func (wm *webmention) receive() { + log.Info().Str("webmention", wm.String()).Msg("OK: looks valid") + body, geterr := common.Get(wm.source) + + if geterr != nil { + log.Warn().Str("source", wm.source).Msg(" ABORT: invalid url") + wm.deletePossibleOlderWebmention() + return + } + + wm.processSourceBody(body) +} + +func (wm *webmention) deletePossibleOlderWebmention() { + +} + +func (wm *webmention) processSourceBody(body string) { + +} diff --git a/app/webmention/validate.go b/app/webmention/validate.go index d02555d..50c6e6d 100644 --- a/app/webmention/validate.go +++ b/app/webmention/validate.go @@ -3,8 +3,11 @@ package webmention import ( "strings" + "net/http" "github.com/wgroeneveld/go-jamming/common" + + "github.com/rs/zerolog/log" ) func isValidUrl(url string) bool { @@ -21,6 +24,7 @@ func isValidDomain(url string, conf *common.Config) bool { return false } +// great, these are needed to do the structural typing for the tests... type httpReq interface { FormValue(key string) string } @@ -28,6 +32,15 @@ type httpHeader interface { Get(key string) string } +func isValidTargetUrl(url string) bool { + _, err := http.Get(url) + if err != nil { + log.Warn().Str("target", url).Msg("Invalid target URL") + return false + } + return true +} + func validate(r httpReq, h httpHeader, conf *common.Config) bool { return h.Get("Content-Type") == "application/x-www-form-urlencoded" && isValidUrl(r.FormValue("source")) && diff --git a/common/httputils.go b/common/httputils.go new file mode 100644 index 0000000..edb7149 --- /dev/null +++ b/common/httputils.go @@ -0,0 +1,37 @@ + +package common + +import ( + "fmt" + "net/http" + "io/ioutil" +) + +func BadRequest(w http.ResponseWriter) { + http.Error(w, "400 bad request", http.StatusBadRequest) +} + +func Accept(w http.ResponseWriter) { + w.WriteHeader(202) + w.Write([]byte("Thanks, bro. Will send these webmentions soon, pinky swear!")) +} + +// something like this? https://freshman.tech/snippets/go/http-response-to-string/ +func Get(url string) (string, error) { + resp, geterr := http.Get(url) + if geterr != nil { + return "", geterr + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return "", fmt.Errorf("Status code for %s is not OK (%d)", url, resp.StatusCode) + } + + defer resp.Body.Close() + body, readerr := ioutil.ReadAll(resp.Body) + if readerr != nil { + return "", readerr + } + + return string(body), nil +} diff --git a/go.mod b/go.mod index c641803..0ad8ea9 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/wgroeneveld/go-jamming go 1.16 -require github.com/gorilla/mux v1.8.0 +require ( + github.com/gorilla/mux v1.8.0 + github.com/rs/zerolog v1.21.0 +) diff --git a/go.sum b/go.sum index 5350288..6d57686 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,29 @@ +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.21.0 h1:Q3vdXlfLNT+OftyBHsU0Y445MD+8m8axjKgf2si0QcM= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/jsfork/src/webmention/receive.js b/jsfork/src/webmention/receive.js index a4ce00c..d90514c 100644 --- a/jsfork/src/webmention/receive.js +++ b/jsfork/src/webmention/receive.js @@ -10,38 +10,8 @@ dayjs.extend(utc) const log = require('pino')() -function isValidUrl(url) { - return url !== undefined && - (url.startsWith("http://") || url.startsWith("https://")) -} - -function isValidDomain(url) { - return config.allowedWebmentionSources.some(domain => { - return url.indexOf(domain) !== -1 - }) -} - -/** -Remember, TARGET is own domain, SOURCE is the article to process - https://www.w3.org/TR/webmention/#sender-notifies-receiver - example: - POST /webmention-endpoint HTTP/1.1 - Host: aaronpk.example - Content-Type: application/x-www-form-urlencoded - - source=https://waterpigs.example/post-by-barnaby& - target=https://aaronpk.example/post-by-aaron - - - HTTP/1.1 202 Accepted -*/ function validate(request) { - return request.type === "application/x-www-form-urlencoded" && - request.body !== undefined && - isValidUrl(request?.body?.source) && - isValidUrl(request?.body?.target) && - request?.body?.source !== request?.body?.target && - isValidDomain(request?.body?.target) + // DONE } async function isValidTargetUrl(target) { diff --git a/main.go b/main.go index fe231bc..89aed1c 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,19 @@ package main import ( + "os" + "github.com/wgroeneveld/go-jamming/app" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) func main() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + // TODO this should only be enabled in local mode. Fix with config? + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + + log.Debug().Msg("Let's a go!") app.Start() } diff --git a/playground.go b/playground.go new file mode 100644 index 0000000..8660ab2 --- /dev/null +++ b/playground.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "io/ioutil" +) + +func mainz() { + fmt.Println("Hello, playground") + resp, err := http.Get("https://brainbaking.com/notes") + if err != nil { + log.Fatalln(err) + } + + body, err2 := ioutil.ReadAll(resp.Body) + if err2 != nil { + log.Fatalln(err) + } + + fmt.Printf("tis ditte") + fmt.Printf("%s", body) +}