diff --git a/app/webmention/load/loader.go b/app/webmention/load/loader.go index 57657ba..58f5e20 100644 --- a/app/webmention/load/loader.go +++ b/app/webmention/load/loader.go @@ -9,6 +9,7 @@ import ( // FromDisk assumes that params have already been validated. func FromDisk(domain string, dataPath string) mf.IndiewebDataResult { loadPath := path.Join(dataPath, domain) + info, _ := ioutil.ReadDir(loadPath) amountOfFiles := len(info) results := make(chan *mf.IndiewebData, amountOfFiles) diff --git a/app/webmention/load/loader_test.go b/app/webmention/load/loader_test.go index 2fbba8d..69e3ccc 100644 --- a/app/webmention/load/loader_test.go +++ b/app/webmention/load/loader_test.go @@ -1,6 +1,7 @@ package load import ( + "fmt" "github.com/stretchr/testify/assert" "io/ioutil" "os" @@ -9,6 +10,40 @@ import ( "testing" ) +// stress tests to see what concurrent disk access is like. Runs fine, even with 5000 runs and 100 files. +// this means worker pools do not have to be implemented in FromDisk(). +// However, if runs := 10000, some results are empty. At other times, even ioutil.ReadDir() panics... +// The rate limiter should catch this. +func TestFromDiskStressTest(t *testing.T) { + runs := 100 + files := 100 + + os.MkdirAll("testdata/somedomain", os.ModePerm) + defer os.RemoveAll("testdata") + + for i := 0; i < files; i++ { + json := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}` + ioutil.WriteFile(fmt.Sprintf("testdata/somedomain/%d.json", i), []byte(json), os.ModePerm) + } + + amounts := make(chan int, runs) + for i := 0; i < runs; i++ { + go func(nr int) { + data := FromDisk("somedomain", "testdata") + itms := len(data.Data) + + fmt.Printf("From disk #%d - found %d items\n", nr, itms) + amounts <- itms + }(i) + } + + fmt.Println("Asserting...") + for i := 0; i < runs; i++ { + actual := <-amounts + assert.Equal(t, files, actual) + } +} + func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) { os.MkdirAll("testdata/somedomain", os.ModePerm) defer os.RemoveAll("testdata") diff --git a/app/webmention/send/send_test.go b/app/webmention/send/send_test.go index e1a2880..0eb164a 100644 --- a/app/webmention/send/send_test.go +++ b/app/webmention/send/send_test.go @@ -4,7 +4,10 @@ import ( "brainbaking.com/go-jamming/app/mf" "brainbaking.com/go-jamming/common" "brainbaking.com/go-jamming/mocks" + "brainbaking.com/go-jamming/rest" + "fmt" "github.com/stretchr/testify/assert" + "net/http" "net/url" "sync" "testing" @@ -30,6 +33,56 @@ func TestSendMentionAsWebmention(t *testing.T) { assert.Equal(t, "mytarget", passedFormValues.Get("target")) } +// Stress test for opening HTTP connections en masse. +// Works great for up to 1000 runs. 10k hits: "http: Accept error: accept tcp [::]:6666: accept: too many open files in system; retrying in 10ms" +// Crashed even GoLand and the open Spotify client... +// The rate limiter fixes this, and in reality, we never send out 10k links anyway. +func TestSendMentionIntegrationStressTest(t *testing.T) { + snder := Sender{ + Conf: common.Configure(), + RestClient: &rest.HttpClient{}, + } + + runs := 100 + responses := make(chan bool, runs) + + http.HandleFunc("/pingback", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(200) + writer.Write([]byte("pingbacked stuff.")) + responses <- true + }) + http.HandleFunc("/target", func(writer http.ResponseWriter, request *http.Request) { + target := ` + + + + sup! + + ` + writer.WriteHeader(200) + writer.Write([]byte(target)) + }) + + go func() { + fmt.Println("Serving stub at 6666...") + http.ListenAndServe(":6666", nil) + fmt.Println("Stub stopped?") + }() + + fmt.Println("Bootstrapping runs...") + for i := 0; i < runs; i++ { + snder.sendMention(mf.Mention{ + Source: "http://localhost:6666/source", + Target: "http://localhost:6666/target", + }) + } + fmt.Println("Asserting...") + for i := 0; i < runs; i++ { + pingbacked := <-responses + assert.True(t, pingbacked) + } +} + func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) { posted := map[string]interface{}{} var lock = sync.RWMutex{}