forked from wgroeneveld/go-jamming
introduced stress tests for paralell parts
This commit is contained in:
parent
cd12b80170
commit
62c55cdbb1
|
@ -9,6 +9,7 @@ import (
|
||||||
// FromDisk assumes that params have already been validated.
|
// FromDisk assumes that params have already been validated.
|
||||||
func FromDisk(domain string, dataPath string) mf.IndiewebDataResult {
|
func FromDisk(domain string, dataPath string) mf.IndiewebDataResult {
|
||||||
loadPath := path.Join(dataPath, domain)
|
loadPath := path.Join(dataPath, domain)
|
||||||
|
|
||||||
info, _ := ioutil.ReadDir(loadPath)
|
info, _ := ioutil.ReadDir(loadPath)
|
||||||
amountOfFiles := len(info)
|
amountOfFiles := len(info)
|
||||||
results := make(chan *mf.IndiewebData, amountOfFiles)
|
results := make(chan *mf.IndiewebData, amountOfFiles)
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package load
|
package load
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
|
@ -9,6 +10,40 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// stress tests to see what concurrent disk access is like. Runs fine, even with 5000 runs and 100 files.
|
||||||
|
// this means worker pools do not have to be implemented in FromDisk().
|
||||||
|
// However, if runs := 10000, some results are empty. At other times, even ioutil.ReadDir() panics...
|
||||||
|
// The rate limiter should catch this.
|
||||||
|
func TestFromDiskStressTest(t *testing.T) {
|
||||||
|
runs := 100
|
||||||
|
files := 100
|
||||||
|
|
||||||
|
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
||||||
|
defer os.RemoveAll("testdata")
|
||||||
|
|
||||||
|
for i := 0; i < files; i++ {
|
||||||
|
json := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
|
||||||
|
ioutil.WriteFile(fmt.Sprintf("testdata/somedomain/%d.json", i), []byte(json), os.ModePerm)
|
||||||
|
}
|
||||||
|
|
||||||
|
amounts := make(chan int, runs)
|
||||||
|
for i := 0; i < runs; i++ {
|
||||||
|
go func(nr int) {
|
||||||
|
data := FromDisk("somedomain", "testdata")
|
||||||
|
itms := len(data.Data)
|
||||||
|
|
||||||
|
fmt.Printf("From disk #%d - found %d items\n", nr, itms)
|
||||||
|
amounts <- itms
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Asserting...")
|
||||||
|
for i := 0; i < runs; i++ {
|
||||||
|
actual := <-amounts
|
||||||
|
assert.Equal(t, files, actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) {
|
func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) {
|
||||||
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
||||||
defer os.RemoveAll("testdata")
|
defer os.RemoveAll("testdata")
|
||||||
|
|
|
@ -4,7 +4,10 @@ import (
|
||||||
"brainbaking.com/go-jamming/app/mf"
|
"brainbaking.com/go-jamming/app/mf"
|
||||||
"brainbaking.com/go-jamming/common"
|
"brainbaking.com/go-jamming/common"
|
||||||
"brainbaking.com/go-jamming/mocks"
|
"brainbaking.com/go-jamming/mocks"
|
||||||
|
"brainbaking.com/go-jamming/rest"
|
||||||
|
"fmt"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
@ -30,6 +33,56 @@ func TestSendMentionAsWebmention(t *testing.T) {
|
||||||
assert.Equal(t, "mytarget", passedFormValues.Get("target"))
|
assert.Equal(t, "mytarget", passedFormValues.Get("target"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stress test for opening HTTP connections en masse.
|
||||||
|
// Works great for up to 1000 runs. 10k hits: "http: Accept error: accept tcp [::]:6666: accept: too many open files in system; retrying in 10ms"
|
||||||
|
// Crashed even GoLand and the open Spotify client...
|
||||||
|
// The rate limiter fixes this, and in reality, we never send out 10k links anyway.
|
||||||
|
func TestSendMentionIntegrationStressTest(t *testing.T) {
|
||||||
|
snder := Sender{
|
||||||
|
Conf: common.Configure(),
|
||||||
|
RestClient: &rest.HttpClient{},
|
||||||
|
}
|
||||||
|
|
||||||
|
runs := 100
|
||||||
|
responses := make(chan bool, runs)
|
||||||
|
|
||||||
|
http.HandleFunc("/pingback", func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
writer.WriteHeader(200)
|
||||||
|
writer.Write([]byte("pingbacked stuff."))
|
||||||
|
responses <- true
|
||||||
|
})
|
||||||
|
http.HandleFunc("/target", func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
target := `<html>
|
||||||
|
<head>
|
||||||
|
<link rel="pingback" href="http://localhost:6666/pingback" />
|
||||||
|
</head>
|
||||||
|
<body>sup!</body>
|
||||||
|
</html>
|
||||||
|
`
|
||||||
|
writer.WriteHeader(200)
|
||||||
|
writer.Write([]byte(target))
|
||||||
|
})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
fmt.Println("Serving stub at 6666...")
|
||||||
|
http.ListenAndServe(":6666", nil)
|
||||||
|
fmt.Println("Stub stopped?")
|
||||||
|
}()
|
||||||
|
|
||||||
|
fmt.Println("Bootstrapping runs...")
|
||||||
|
for i := 0; i < runs; i++ {
|
||||||
|
snder.sendMention(mf.Mention{
|
||||||
|
Source: "http://localhost:6666/source",
|
||||||
|
Target: "http://localhost:6666/target",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fmt.Println("Asserting...")
|
||||||
|
for i := 0; i < runs; i++ {
|
||||||
|
pingbacked := <-responses
|
||||||
|
assert.True(t, pingbacked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
|
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
|
||||||
posted := map[string]interface{}{}
|
posted := map[string]interface{}{}
|
||||||
var lock = sync.RWMutex{}
|
var lock = sync.RWMutex{}
|
||||||
|
|
Loading…
Reference in New Issue