From 824c64c93378cdd6a455fc736805e9273ee1360d Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Sun, 11 Apr 2021 13:03:41 +0200 Subject: [PATCH] scatter/gather in json loading --- app/mf/microformats.go | 12 +++++++++ app/server.go | 8 ++---- app/webmention/handler.go | 7 +++-- app/webmention/load/loader.go | 41 ++++++++++++++++++++++++++++++ app/webmention/load/loader_test.go | 33 ++++++++++++++++++++++++ rest/utils.go | 12 +++++++++ 6 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 app/webmention/load/loader.go create mode 100644 app/webmention/load/loader_test.go diff --git a/app/mf/microformats.go b/app/mf/microformats.go index 1eb7a2b..77b5a00 100644 --- a/app/mf/microformats.go +++ b/app/mf/microformats.go @@ -16,6 +16,18 @@ type IndiewebAuthor struct { Picture string `json:"picture"` } +type IndiewebDataResult struct { + Status string `json:"status"` + Data []*IndiewebData `json:"json"` +} + +func WrapResult(data []*IndiewebData) IndiewebDataResult { + return IndiewebDataResult{ + Status: "success", + Data: data, + } +} + type IndiewebData struct { Author IndiewebAuthor `json:"author"` Name string `json:"name"` diff --git a/app/server.go b/app/server.go index e9b4cc5..5218f57 100644 --- a/app/server.go +++ b/app/server.go @@ -1,6 +1,7 @@ package app import ( + "brainbaking.com/go-jamming/rest" "net/http" "strconv" @@ -15,16 +16,11 @@ type server struct { conf *common.Config } -// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076 -func unauthorized(w http.ResponseWriter, r *http.Request) { - http.Error(w, "401 unauthorized", http.StatusUnauthorized) -} - func (s *server) authorizedOnly(h http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) if vars["token"] != s.conf.Token || !s.conf.IsAnAllowedDomain(vars["domain"]) { - unauthorized(w, r) + rest.Unauthorized(w) return } h(w, r) diff --git a/app/webmention/handler.go b/app/webmention/handler.go index 36a0f44..b0f76a5 100644 --- a/app/webmention/handler.go +++ b/app/webmention/handler.go @@ -2,9 +2,9 @@ package webmention import ( "brainbaking.com/go-jamming/app/mf" + "brainbaking.com/go-jamming/app/webmention/load" "brainbaking.com/go-jamming/app/webmention/recv" "brainbaking.com/go-jamming/app/webmention/send" - "fmt" "github.com/gorilla/mux" "net/http" @@ -16,7 +16,10 @@ var httpClient = &rest.HttpClient{} func HandleGet(conf *common.Config) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - fmt.Println("handling get") + domain := mux.Vars(r)["domain"] + result := load.FromDisk(domain, conf.DataPath) + + rest.Json(w, result) } } diff --git a/app/webmention/load/loader.go b/app/webmention/load/loader.go new file mode 100644 index 0000000..4c47b1c --- /dev/null +++ b/app/webmention/load/loader.go @@ -0,0 +1,41 @@ +package load + +import ( + "brainbaking.com/go-jamming/app/mf" + "encoding/json" + "io/ioutil" + "path" +) + +func FromDisk(domain string, dataPath string) mf.IndiewebDataResult { + // assume that params have already been validated. + loadPath := path.Join(dataPath, domain) + info, _ := ioutil.ReadDir(loadPath) + amountOfFiles := len(info) + results := make(chan *mf.IndiewebData, amountOfFiles) + + for _, file := range info { + fileName := file.Name() + go func() { + data, _ := ioutil.ReadFile(path.Join(loadPath, fileName)) + indiewebData := &mf.IndiewebData{} + json.Unmarshal(data, indiewebData) + results <- indiewebData + }() + } + + indiewebResults := gather(amountOfFiles, results) + return mf.WrapResult(indiewebResults) +} + +func gather(amount int, results chan *mf.IndiewebData) []*mf.IndiewebData { + var indiewebResults []*mf.IndiewebData + for i := 0; i < amount; i++ { + result := <-results + // json marshal errors are ignored in the above scatter func.Highly unlikely, but still. + if result.Url != "" { + indiewebResults = append(indiewebResults, result) + } + } + return indiewebResults +} diff --git a/app/webmention/load/loader_test.go b/app/webmention/load/loader_test.go new file mode 100644 index 0000000..2fbba8d --- /dev/null +++ b/app/webmention/load/loader_test.go @@ -0,0 +1,33 @@ +package load + +import ( + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "sort" + "strings" + "testing" +) + +func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) { + os.MkdirAll("testdata/somedomain", os.ModePerm) + defer os.RemoveAll("testdata") + + json1 := `{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}` + json2 := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}` + ioutil.WriteFile("testdata/somedomain/testjson1.json", []byte(json1), os.ModePerm) + ioutil.WriteFile("testdata/somedomain/testjson2.json", []byte(json2), os.ModePerm) + + result := FromDisk("somedomain", "testdata") + sort.SliceStable(result.Data, func(i, j int) bool { + comp := strings.Compare(result.Data[i].Author.Name, result.Data[j].Author.Name) + if comp > 0 { + return false + } + return true + }) + + assert.Equal(t, "success", result.Status) + assert.Equal(t, "Jef Klakveld", result.Data[0].Author.Name) + assert.Equal(t, "Wouter Groeneveld", result.Data[1].Author.Name) +} diff --git a/rest/utils.go b/rest/utils.go index 637e445..2e6ff8b 100644 --- a/rest/utils.go +++ b/rest/utils.go @@ -1,14 +1,26 @@ package rest import ( + "encoding/json" "net/http" "net/url" ) +// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076 func BadRequest(w http.ResponseWriter) { http.Error(w, "400 bad request", http.StatusBadRequest) } +func Unauthorized(w http.ResponseWriter) { + http.Error(w, "401 unauthorized", http.StatusUnauthorized) +} + +func Json(w http.ResponseWriter, data interface{}) { + w.WriteHeader(200) + bytes, _ := json.Marshal(data) + w.Write(bytes) +} + func Accept(w http.ResponseWriter) { w.WriteHeader(202) w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!"))