scatter/gather in json loading

This commit is contained in:
Wouter Groeneveld 2021-04-11 13:03:41 +02:00
parent e194293a46
commit 824c64c933
6 changed files with 105 additions and 8 deletions

View File

@ -16,6 +16,18 @@ type IndiewebAuthor struct {
Picture string `json:"picture"`
}
type IndiewebDataResult struct {
Status string `json:"status"`
Data []*IndiewebData `json:"json"`
}
func WrapResult(data []*IndiewebData) IndiewebDataResult {
return IndiewebDataResult{
Status: "success",
Data: data,
}
}
type IndiewebData struct {
Author IndiewebAuthor `json:"author"`
Name string `json:"name"`

View File

@ -1,6 +1,7 @@
package app
import (
"brainbaking.com/go-jamming/rest"
"net/http"
"strconv"
@ -15,16 +16,11 @@ type server struct {
conf *common.Config
}
// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076
func unauthorized(w http.ResponseWriter, r *http.Request) {
http.Error(w, "401 unauthorized", http.StatusUnauthorized)
}
func (s *server) authorizedOnly(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
if vars["token"] != s.conf.Token || !s.conf.IsAnAllowedDomain(vars["domain"]) {
unauthorized(w, r)
rest.Unauthorized(w)
return
}
h(w, r)

View File

@ -2,9 +2,9 @@ package webmention
import (
"brainbaking.com/go-jamming/app/mf"
"brainbaking.com/go-jamming/app/webmention/load"
"brainbaking.com/go-jamming/app/webmention/recv"
"brainbaking.com/go-jamming/app/webmention/send"
"fmt"
"github.com/gorilla/mux"
"net/http"
@ -16,7 +16,10 @@ var httpClient = &rest.HttpClient{}
func HandleGet(conf *common.Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Println("handling get")
domain := mux.Vars(r)["domain"]
result := load.FromDisk(domain, conf.DataPath)
rest.Json(w, result)
}
}

View File

@ -0,0 +1,41 @@
package load
import (
"brainbaking.com/go-jamming/app/mf"
"encoding/json"
"io/ioutil"
"path"
)
func FromDisk(domain string, dataPath string) mf.IndiewebDataResult {
// assume that params have already been validated.
loadPath := path.Join(dataPath, domain)
info, _ := ioutil.ReadDir(loadPath)
amountOfFiles := len(info)
results := make(chan *mf.IndiewebData, amountOfFiles)
for _, file := range info {
fileName := file.Name()
go func() {
data, _ := ioutil.ReadFile(path.Join(loadPath, fileName))
indiewebData := &mf.IndiewebData{}
json.Unmarshal(data, indiewebData)
results <- indiewebData
}()
}
indiewebResults := gather(amountOfFiles, results)
return mf.WrapResult(indiewebResults)
}
func gather(amount int, results chan *mf.IndiewebData) []*mf.IndiewebData {
var indiewebResults []*mf.IndiewebData
for i := 0; i < amount; i++ {
result := <-results
// json marshal errors are ignored in the above scatter func.Highly unlikely, but still.
if result.Url != "" {
indiewebResults = append(indiewebResults, result)
}
}
return indiewebResults
}

View File

@ -0,0 +1,33 @@
package load
import (
"github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
)
func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) {
os.MkdirAll("testdata/somedomain", os.ModePerm)
defer os.RemoveAll("testdata")
json1 := `{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Manias Lock On to Belgiums third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
json2 := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Manias Lock On to Belgiums third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
ioutil.WriteFile("testdata/somedomain/testjson1.json", []byte(json1), os.ModePerm)
ioutil.WriteFile("testdata/somedomain/testjson2.json", []byte(json2), os.ModePerm)
result := FromDisk("somedomain", "testdata")
sort.SliceStable(result.Data, func(i, j int) bool {
comp := strings.Compare(result.Data[i].Author.Name, result.Data[j].Author.Name)
if comp > 0 {
return false
}
return true
})
assert.Equal(t, "success", result.Status)
assert.Equal(t, "Jef Klakveld", result.Data[0].Author.Name)
assert.Equal(t, "Wouter Groeneveld", result.Data[1].Author.Name)
}

View File

@ -1,14 +1,26 @@
package rest
import (
"encoding/json"
"net/http"
"net/url"
)
// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076
func BadRequest(w http.ResponseWriter) {
http.Error(w, "400 bad request", http.StatusBadRequest)
}
func Unauthorized(w http.ResponseWriter) {
http.Error(w, "401 unauthorized", http.StatusUnauthorized)
}
func Json(w http.ResponseWriter, data interface{}) {
w.WriteHeader(200)
bytes, _ := json.Marshal(data)
w.Write(bytes)
}
func Accept(w http.ResponseWriter) {
w.WriteHeader(202)
w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!"))