emeraldpay / emeraldpay/dshackle
Out of Memory Errors with large debug_traceTransaction
- Dominant language
- Rust
- Stars
- 350
- Forks
- 57
- PR merge metrics
- No merged PRs in 30d
Description
Sometimes, but not always, `dshackle` will crash with out of memory errors dealing with large replies.
```
2021-09-01 16:56:14.243 | WARN | hannelHandlerContext | An exception 'java.lang.OutOfMemoryError: Cannot reserve 16777216 bytes of direct buffer memory (allocated: 1544208392, limit: 1560281088)' [enable DEBUG level for full stacktrace] was thrown by a user handler's exceptionCaught() method while handling the following exception:
at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:768) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:227) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.CompositeByteBuf.addComponents(CompositeByteBuf.java:444) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at reactor.netty.ByteBufFlux.lambda$aggregate$7(ByteBufFlux.java:271) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at reactor.netty.channel.ChannelOperations.terminate(ChannelOperations.java:423) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at reactor.core.publisher.MonoCollectList$MonoCollectListSubscriber.onComplete(MonoCollectList.java:121) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:96) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]
2021-09-01 16:56:14.239 | ERROR | nelOperationsHandler | [id: 0xdbe38e68, L:/172.22.0.3:60014 - R:archive01.archivenode.io/104.227.245.82:8545] Error was received while reading the incoming data. The connection will be closed.
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.CompositeByteBuf.allocBuffer(CompositeByteBuf.java:1751) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:796) [netty-transport-native-epoll-4.1.34.Final-linux-x86_64.jar:4.1.34.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) [netty-transport-native-epoll-4.1.34.Final-linux-x86_64.jar:4.1.34.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.49.Final.jar:4.1.49.Final]
java.lang.OutOfMemoryError: Cannot reserve 16777216 bytes of direct buffer memory (allocated: 1544208392, limit: 1560281088)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]
at java.nio.Bits.reserveMemory(Bits.java:178) ~[?:?]
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:327) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:147) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.CompositeByteBuf.consolidateIfNeeded(CompositeByteBuf.java:459) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.CompositeByteBuf.allocBuffer(CompositeByteBuf.java:1751) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at reactor.core.publisher.FluxHandleFuseable$HandleFuseableSubscriber.onNext(FluxHandleFuseable.java:163) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:178) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at reactor.core.publisher.Operators$MonoSubscriber.complete(Operators.java:1755) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at reactor.netty.channel.ChannelOperations.onInboundComplete(ChannelOperations.java:367) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:607) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.34.Final.jar:4.1.34.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:427) [netty-transport-native-epoll-4.1.34.Final-linux-x86_64.jar:4.1.34.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) [netty-transport-native-epoll-4.1.34.Final-linux-x86_64.jar:4.1.34.Final]
at java.lang.Thread.run(Thread.java:830) [?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]
at java.nio.Bits.reserveMemory(Bits.java:178) ~[?:?]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:227) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.PoolArena.allocate(PoolArena.java:147) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at reactor.core.publisher.MonoCollectList$MonoCollectListSubscriber.onComplete(MonoCollectList.java:121) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at reactor.netty.channel.FluxReceive.onInboundComplete(FluxReceive.java:366) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:607) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.34.Final.jar:4.1.34.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at java.lang.Thread.run(Thread.java:830) [?:?]
at java.nio.DirectByteBuffer.(DirectByteBuffer.java:119) ~[?:?]
at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:245) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:744) ~[netty-buffer-4.1.34.Final.jar:4.1.34.Final]
at reactor.core.publisher.FluxContextStart$ContextStartSubscriber.onNext(FluxContextStart.java:96) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at reactor.netty.channel.FluxReceive.onInboundComplete(FluxReceive.java:366) ~[reactor-netty-0.9.7.RELEASE.jar:0.9.7.RELEASE]
at reactor.core.publisher.FluxMap$MapSubscriber.onComplete(FluxMap.java:136) ~[reactor-core-3.3.5.RELEASE.jar:3.3.5.RELEASE]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.34.Final.jar:4.1.34.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]
```
The output from the node is just under 4MB in size
`curl -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"debug_traceTransaction","params": ["0x391aa8c844664ea4ab596ee1ce83fb0b7053ecfad138386073d72304aa6c8cec"],"id":73}'`
The docker host itself seems to have plenty of RAM...
```
mysticryuujin@docker01:~$ free -h
total used free shared buff/cache available
Mem: 5.8Gi 4.2Gi 354Mi 7.0Mi 1.2Gi 1.3Gi
Swap: 2.0Gi 3.0Mi 2.0Gi
```
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.