forked from wgroeneveld/go-jamming
introduced stress tests for paralell parts
This commit is contained in:
parent
cd12b80170
commit
62c55cdbb1
|
|
@ -9,6 +9,7 @@ import (
|
|||
// FromDisk assumes that params have already been validated.
|
||||
func FromDisk(domain string, dataPath string) mf.IndiewebDataResult {
|
||||
loadPath := path.Join(dataPath, domain)
|
||||
|
||||
info, _ := ioutil.ReadDir(loadPath)
|
||||
amountOfFiles := len(info)
|
||||
results := make(chan *mf.IndiewebData, amountOfFiles)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package load
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
|
@ -9,6 +10,40 @@ import (
|
|||
"testing"
|
||||
)
|
||||
|
||||
// stress tests to see what concurrent disk access is like. Runs fine, even with 5000 runs and 100 files.
|
||||
// this means worker pools do not have to be implemented in FromDisk().
|
||||
// However, if runs := 10000, some results are empty. At other times, even ioutil.ReadDir() panics...
|
||||
// The rate limiter should catch this.
|
||||
func TestFromDiskStressTest(t *testing.T) {
|
||||
runs := 100
|
||||
files := 100
|
||||
|
||||
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
for i := 0; i < files; i++ {
|
||||
json := `{"author":{"name":"Jef Klakveld","picture":"https://brainbaking.com/img/avatar.jpg"},"name":"I much prefer Sonic Mania's Lock On to Belgium's t...","content":"I much prefer Sonic Mania’s Lock On to Belgium’s third Lock Down. Sigh. At least 16-bit 2D platformers make me smile: https://jefklakscodex.com/articles/reviews/sonic-mania/\n\n\n\nEnclosed Toot image","published":"2021-03-25T10:45:00","url":"https://brainbaking.com/notes/2021/03/25h10m45s09/","type":"mention","source":"https://brainbaking.com/notes/2021/03/25h10m45s09/","target":"https://jefklakscodex.com/articles/reviews/sonic-mania/"}`
|
||||
ioutil.WriteFile(fmt.Sprintf("testdata/somedomain/%d.json", i), []byte(json), os.ModePerm)
|
||||
}
|
||||
|
||||
amounts := make(chan int, runs)
|
||||
for i := 0; i < runs; i++ {
|
||||
go func(nr int) {
|
||||
data := FromDisk("somedomain", "testdata")
|
||||
itms := len(data.Data)
|
||||
|
||||
fmt.Printf("From disk #%d - found %d items\n", nr, itms)
|
||||
amounts <- itms
|
||||
}(i)
|
||||
}
|
||||
|
||||
fmt.Println("Asserting...")
|
||||
for i := 0; i < runs; i++ {
|
||||
actual := <-amounts
|
||||
assert.Equal(t, files, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromDiskReturnsAllJsonFilesFromDiskWrappedInResult(t *testing.T) {
|
||||
os.MkdirAll("testdata/somedomain", os.ModePerm)
|
||||
defer os.RemoveAll("testdata")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@ import (
|
|||
"brainbaking.com/go-jamming/app/mf"
|
||||
"brainbaking.com/go-jamming/common"
|
||||
"brainbaking.com/go-jamming/mocks"
|
||||
"brainbaking.com/go-jamming/rest"
|
||||
"fmt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
|
|
@ -30,6 +33,56 @@ func TestSendMentionAsWebmention(t *testing.T) {
|
|||
assert.Equal(t, "mytarget", passedFormValues.Get("target"))
|
||||
}
|
||||
|
||||
// Stress test for opening HTTP connections en masse.
|
||||
// Works great for up to 1000 runs. 10k hits: "http: Accept error: accept tcp [::]:6666: accept: too many open files in system; retrying in 10ms"
|
||||
// Crashed even GoLand and the open Spotify client...
|
||||
// The rate limiter fixes this, and in reality, we never send out 10k links anyway.
|
||||
func TestSendMentionIntegrationStressTest(t *testing.T) {
|
||||
snder := Sender{
|
||||
Conf: common.Configure(),
|
||||
RestClient: &rest.HttpClient{},
|
||||
}
|
||||
|
||||
runs := 100
|
||||
responses := make(chan bool, runs)
|
||||
|
||||
http.HandleFunc("/pingback", func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.WriteHeader(200)
|
||||
writer.Write([]byte("pingbacked stuff."))
|
||||
responses <- true
|
||||
})
|
||||
http.HandleFunc("/target", func(writer http.ResponseWriter, request *http.Request) {
|
||||
target := `<html>
|
||||
<head>
|
||||
<link rel="pingback" href="http://localhost:6666/pingback" />
|
||||
</head>
|
||||
<body>sup!</body>
|
||||
</html>
|
||||
`
|
||||
writer.WriteHeader(200)
|
||||
writer.Write([]byte(target))
|
||||
})
|
||||
|
||||
go func() {
|
||||
fmt.Println("Serving stub at 6666...")
|
||||
http.ListenAndServe(":6666", nil)
|
||||
fmt.Println("Stub stopped?")
|
||||
}()
|
||||
|
||||
fmt.Println("Bootstrapping runs...")
|
||||
for i := 0; i < runs; i++ {
|
||||
snder.sendMention(mf.Mention{
|
||||
Source: "http://localhost:6666/source",
|
||||
Target: "http://localhost:6666/target",
|
||||
})
|
||||
}
|
||||
fmt.Println("Asserting...")
|
||||
for i := 0; i < runs; i++ {
|
||||
pingbacked := <-responses
|
||||
assert.True(t, pingbacked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendIntegrationTestCanSendBothWebmentionsAndPingbacks(t *testing.T) {
|
||||
posted := map[string]interface{}{}
|
||||
var lock = sync.RWMutex{}
|
||||
|
|
|
|||
Loading…
Reference in New Issue