ChainSafe / ChainSafe/gossamer

Stress test `TestSync_Bench` times out or deadlocks

Open
#2,304 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Go
Stars
454
Forks
144
PR merge metrics
No merged PRs in 30d

Description

[Logs](https://github.com/ChainSafe/gossamer/runs/5219155830?check_suite_focus=true), [other logs](https://github.com/ChainSafe/gossamer/actions/runs/1859914208/attempts/2)

```
=== RUN TestSync_Bench
2022-02-16T16:22:13Z INFO initialising gossamer using /go/src/github.com/ChainSafe/gossamer/bin/gossamer init --config /go/src/github.com/ChainSafe/gossamer/tests/utils/config_nograndpa.toml --basepath /tmp/TestSync_Bench/bob --genesis /go/src/github.com/ChainSafe/gossamer/chain/dev/genesis-spec.json --force... pkg=test/utils
2022-02-16T16:22:17Z INFO initialised gossamer node 0! pkg=test/utils
2022-02-16T16:22:17Z INFO starting gossamer at /go/src/github.com/ChainSafe/gossamer/bin/gossamer --port 7000 --config /go/src/github.com/ChainSafe/gossamer/tests/utils/config_nograndpa.toml --basepath /tmp/TestSync_Bench/bob --rpchost 0.0.0.0 --rpcport 8540 --rpcmods system,author,chain,state,dev,rpc --rpc --no-telemetry --log info --babe-lead --roles 4 --key alice... pkg=test/utils
2022-02-16T16:22:27Z INFO node started with key alice and cmd.Process.Pid 5732 pkg=test/utils
panic: test timed out after 15m0s

goroutine 2285 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:1788 +0x8e
created by time.goFunc
/usr/local/go/src/time/sleep.go:180 +0x31

goroutine 1 [chan receive, 2 minutes]:
testing.(*T).Run(0xc00008a820, {0xe7c17a, 0x47[62](https://github.com/ChainSafe/gossamer/runs/5219155830?check_suite_focus=true#step:5:62)53}, 0x10aa850)
/usr/local/go/src/testing/testing.go:1307 +0x375
testing.runTests.func1(0xc0017360f0)
/usr/local/go/src/testing/testing.go:1598 +0x6e
testing.tRunner(0xc00008a820, 0xc0002f7c48)
/usr/local/go/src/testing/testing.go:1259 +0x102
testing.runTests(0xc00009e180, {0x192f9e0, 0x10, 0x10}, {0x4b43de, 0xcffd00, 0x1b8cd40})
/usr/local/go/src/testing/testing.go:1596 +0x43f
testing.(*M).Run(0xc00009e180)
/usr/local/go/src/testing/testing.go:1504 +0x51d
github.com/ChainSafe/gossamer/tests/stress.TestMain(0xc0000001a0)
/go/src/github.com/ChainSafe/gossamer/tests/stress/stress_test.go:65 +0x254
main.main()
_testmain.go:75 +0x165

goroutine 10 [select, 15 minutes]:
github.com/ipfs/go-log/writer.(*MirrorWriter).logRoutine(0xc0002c2780)
/go/pkg/mod/github.com/ipfs/go-log@v1.0.5/writer/writer.go:71 +0x110
created by github.com/ipfs/go-log/writer.NewMirrorWriter
/go/pkg/mod/github.com/ipfs/go-log@v1.0.5/writer/writer.go:36 +0xcf

goroutine 19 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00009e080)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92

goroutine 2284 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7fac2a9ac9f8, 0x72)
/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc001976420, 0xc0015ac000, 0x1)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001976420, {0xc0015ac000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
os.(*File).read(...)
/usr/local/go/src/os/file_posix.go:32
os.(*File).Read(0xc000ffc160, {0xc0015ac000, 0xc001913270, 0xc001913270})
/usr/local/go/src/os/file.go:119 +0x5e
io.copyBuffer({0x1187ba0, 0xc001913270}, {0x1186800, 0xc000ffc160}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:423 +0x1b2
io.Copy(...)
/usr/local/go/src/io/io.go:382
os.genericReadFrom(0xc000038ba0, {0x1186800, 0xc000ffc160})
/usr/local/go/src/os/file.go:162 +0x5d
os.(*File).ReadFrom(0xc000ffc140, {0x1186800, 0xc000ffc160})
/usr/local/go/src/os/file.go:156 +0x1b0
bufio.(*Writer).ReadFrom(0xc001844840, {0x1186800, 0xc000ffc160})
/usr/local/go/src/bufio/bufio.go:731 +0xa6
io.copyBuffer({0x11853a0, 0xc001844840}, {0x1186800, 0xc000ffc160}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:409 +0x14b
io.Copy(...)
/usr/local/go/src/io/io.go:382
github.com/ChainSafe/gossamer/tests/utils.startGossamer.func3()
/go/src/github.com/ChainSafe/gossamer/tests/utils/gossamer_utils.go:188 +0x54
created by github.com/ChainSafe/gossamer/tests/utils.startGossamer
/go/src/github.com/ChainSafe/gossamer/tests/utils/gossamer_utils.go:187 +0xc0f

goroutine 2281 [sleep]:
time.Sleep(0xb2d05e00)
/usr/local/go/src/runtime/time.go:193 +0x12e
github.com/ChainSafe/gossamer/tests/stress.TestSync_Bench(0xc0010896c0)
/go/src/github.com/ChainSafe/gossamer/tests/stress/stress_test.go:228 +0x208
testing.tRunner(0xc0010896c0, 0x10aa850)
/usr/local/go/src/testing/testing.go:1259 +0x102
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1306 +0x35a

goroutine 2283 [IO wait]:
internal/poll.runtime_pollWait(0x7fac2a9ac740, 0x72)
/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00197[63](https://github.com/ChainSafe/gossamer/runs/5219155830?check_suite_focus=true#step:5:63)60, 0xc0015b4000, 0x1)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001976360, {0xc0015b4000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
os.(*File).read(...)
/usr/local/go/src/os/file_posix.go:32
os.(*File).Read(0xc000ffc150, {0xc0015b4000, 0xc001913280, 0xc001913280})
/usr/local/go/src/os/file.go:119 +0x5e
io.copyBuffer({0x1187ba0, 0xc001913280}, {0x1186800, 0xc000ffc150}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:423 +0x1b2
io.Copy(...)
/usr/local/go/src/io/io.go:382
os.genericReadFrom(0x46fb7d, {0x1186800, 0xc000ffc150})
/usr/local/go/src/os/file.go:162 +0x5d
os.(*File).ReadFrom(0xc000ffc138, {0x1186800, 0xc000ffc150})
/usr/local/go/src/os/file.go:156 +0x1b0
bufio.(*Writer).ReadFrom(0xc001844800, {0x1186800, 0xc000ffc150})
/usr/local/go/src/bufio/bufio.go:731 +0xa6
io.copyBuffer({0x11853a0, 0xc001844800}, {0x1186800, 0xc000ffc150}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:409 +0x14b
io.Copy(...)
/usr/local/go/src/io/io.go:382
github.com/ChainSafe/gossamer/tests/utils.startGossamer.func2()
/go/src/github.com/ChainSafe/gossamer/tests/utils/gossamer_utils.go:181 +0x54
created by github.com/ChainSafe/gossamer/tests/utils.startGossamer
/go/src/github.com/ChainSafe/gossamer/tests/utils/gossamer_utils.go:180 +0xaef

goroutine 2256 [IO wait]:
internal/poll.runtime_pollWait(0x7fac2a9ac828, 0x72)
/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00019c880, 0xc000ca6000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00019c880, {0xc000ca6000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00019c880, {0xc000ca6000, 0x442827, 0xc00005ec30})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000f8108, {0xc000ca6000, 0xc, 0xc00008ad00})
/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc000015e60, {0xc000ca6000, 0xc00198d860, 0xc00005ed30})
/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0019ccfc0)
/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0019ccfc0, 0x1)
/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc000015e60)
/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2257 [select]:
net/http.(*persistConn).writeLoop(0xc000015e60)
/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1748 +0x1e[65](https://github.com/ChainSafe/gossamer/runs/5219155830?check_suite_focus=true#step:5:65)
FAIL github.com/ChainSafe/gossamer/tests/stress 900.2[70](https://github.com/ChainSafe/gossamer/runs/5219155830?check_suite_focus=true#step:5:70)s
FAIL
make: *** [Makefile:50: it-stress] Error 1
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.