scatter/gather in json loading
This commit is contained in:
parent
e194293a46
commit
824c64c933
|
@ -16,6 +16,18 @@ type IndiewebAuthor struct {
|
||||||
Picture string `json:"picture"`
|
Picture string `json:"picture"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type IndiewebDataResult struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Data []*IndiewebData `json:"json"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func WrapResult(data []*IndiewebData) IndiewebDataResult {
|
||||||
|
return IndiewebDataResult{
|
||||||
|
Status: "success",
|
||||||
|
Data: data,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type IndiewebData struct {
|
type IndiewebData struct {
|
||||||
Author IndiewebAuthor `json:"author"`
|
Author IndiewebAuthor `json:"author"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"brainbaking.com/go-jamming/rest"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
|
@ -15,16 +16,11 @@ type server struct {
|
||||||
conf *common.Config
|
conf *common.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076
|
|
||||||
func unauthorized(w http.ResponseWriter, r *http.Request) {
|
|
||||||
http.Error(w, "401 unauthorized", http.StatusUnauthorized)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *server) authorizedOnly(h http.HandlerFunc) http.HandlerFunc {
|
func (s *server) authorizedOnly(h http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
if vars["token"] != s.conf.Token || !s.conf.IsAnAllowedDomain(vars["domain"]) {
|
if vars["token"] != s.conf.Token || !s.conf.IsAnAllowedDomain(vars["domain"]) {
|
||||||
unauthorized(w, r)
|
rest.Unauthorized(w)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
h(w, r)
|
h(w, r)
|
||||||
|
|
|
@ -2,9 +2,9 @@ package webmention
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"brainbaking.com/go-jamming/app/mf"
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
|
"brainbaking.com/go-jamming/app/webmention/load"
|
||||||
"brainbaking.com/go-jamming/app/webmention/recv"
|
"brainbaking.com/go-jamming/app/webmention/recv"
|
||||||
"brainbaking.com/go-jamming/app/webmention/send"
|
"brainbaking.com/go-jamming/app/webmention/send"
|
||||||
"fmt"
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
@ -16,7 +16,10 @@ var httpClient = &rest.HttpClient{}
|
||||||
|
|
||||||
func HandleGet(conf *common.Config) http.HandlerFunc {
|
func HandleGet(conf *common.Config) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
fmt.Println("handling get")
|
domain := mux.Vars(r)["domain"]
|
||||||
|
result := load.FromDisk(domain, conf.DataPath)
|
||||||
|
|
||||||
|
rest.Json(w, result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,41 @@
|
||||||
|
package load
|
||||||
|
|
||||||
|
import (
|
||||||
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
|
"encoding/json"
|
||||||
|
"io/ioutil"
|
||||||
|
"path"
|
||||||
|
)
|
||||||
|
|
||||||
|
func FromDisk(domain string, dataPath string) mf.IndiewebDataResult {
|
||||||
|
// assume that params have already been validated.
|
||||||
|
loadPath := path.Join(dataPath, domain)
|
||||||
|
info, _ := ioutil.ReadDir(loadPath)
|
||||||
|
amountOfFiles := len(info)
|
||||||
|
results := make(chan *mf.IndiewebData, amountOfFiles)
|
||||||
|
|
||||||
|
for _, file := range info {
|
||||||
|
fileName := file.Name()
|
||||||
|
go func() {
|
||||||
|
data, _ := ioutil.ReadFile(path.Join(loadPath, fileName))
|
||||||
|
indiewebData := &mf.IndiewebData{}
|
||||||
|
json.Unmarshal(data, indiewebData)
|
||||||
|
results <- indiewebData
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
indiewebResults := gather(amountOfFiles, results)
|
||||||
|
return mf.WrapResult(indiewebResults)
|
||||||
|
}
|
||||||
|
|
||||||
|
func gather(amount int, results chan *mf.IndiewebData) []*mf.IndiewebData {
|
||||||
|
var indiewebResults []*mf.IndiewebData
|
||||||
|
for i := 0; i < amount; i++ {
|
||||||
|
result := <-results
|
||||||
|
// json marshal errors are ignored in the above scatter func.Highly unlikely, but still.
|
||||||
|
if result.Url != "" {
|
||||||
|
indiewebResults = append(indiewebResults, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return indiewebResults
|
||||||
|
}
|
|
@ -0,0 +1,33 @@
|
||||||
|
package load
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) {
|
||||||
|
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
||||||
|
defer os.RemoveAll("testdata")
|
||||||
|
|
||||||
|
json1 := `{"author":{"name":"Wouter Groeneveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
|
||||||
|
json2 := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
|
||||||
|
ioutil.WriteFile("testdata/somedomain/testjson1.json", []byte(json1), os.ModePerm)
|
||||||
|
ioutil.WriteFile("testdata/somedomain/testjson2.json", []byte(json2), os.ModePerm)
|
||||||
|
|
||||||
|
result := FromDisk("somedomain", "testdata")
|
||||||
|
sort.SliceStable(result.Data, func(i, j int) bool {
|
||||||
|
comp := strings.Compare(result.Data[i].Author.Name, result.Data[j].Author.Name)
|
||||||
|
if comp > 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.Equal(t, "success", result.Status)
|
||||||
|
assert.Equal(t, "Jef Klakveld", result.Data[0].Author.Name)
|
||||||
|
assert.Equal(t, "Wouter Groeneveld", result.Data[1].Author.Name)
|
||||||
|
}
|
|
@ -1,14 +1,26 @@
|
||||||
package rest
|
package rest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// mimicing NotFound: https://golang.org/src/net/http/server.go?s=64787:64830#L2076
|
||||||
func BadRequest(w http.ResponseWriter) {
|
func BadRequest(w http.ResponseWriter) {
|
||||||
http.Error(w, "400 bad request", http.StatusBadRequest)
|
http.Error(w, "400 bad request", http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Unauthorized(w http.ResponseWriter) {
|
||||||
|
http.Error(w, "401 unauthorized", http.StatusUnauthorized)
|
||||||
|
}
|
||||||
|
|
||||||
|
func Json(w http.ResponseWriter, data interface{}) {
|
||||||
|
w.WriteHeader(200)
|
||||||
|
bytes, _ := json.Marshal(data)
|
||||||
|
w.Write(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
func Accept(w http.ResponseWriter) {
|
func Accept(w http.ResponseWriter) {
|
||||||
w.WriteHeader(202)
|
w.WriteHeader(202)
|
||||||
w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!"))
|
w.Write([]byte("Thanks, bro. Will process this soon, pinky swear!"))
|
||||||
|
|
Loading…
Reference in New Issue