Performance: Netty HTTP/1.1 and HTTPS/2 server in one or two cpu Docker container

Expected behavior

PING app (172.19.0.2) 56(84) bytes of data.
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=1 ttl=64 time=0.145 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=2 ttl=64 time=0.108 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=3 ttl=64 time=0.132 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=4 ttl=64 time=0.106 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=5 ttl=64 time=0.100 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=6 ttl=64 time=0.124 ms
64 bytes from app-netty.netty_default (172.19.0.2): icmp_seq=7 ttl=64 time=0.111 ms

I would have expected 95 and 99 percentile latency to be around 600 microsecond, and always less than 1ms.

Actual behavior

Running multiple tests shows similar latency when running Netty server with one cpu, and EPoll available.

Latencies     [mean, 50, 95, 99, max]  1.101777ms, 978.305µs, 1.661606ms, 3.134684ms, 15.771632ms

Steps to reproduce

I have a simple microservice that exposes /health end point, and always returns the below response. JSON output is stored as bytes, to remove any serialization cost.

{"healthCheck":"OK"}

My test set up is

• application in a docker container. limit cpu resource count to 1 and memory to 1 GB
• vegeta HTTP load tester in another container. limit cpu resource count to 1 and memory to 300 MB. Test run script below.

for i in {1..5}
do
   echo "====="
   echo "`date`: Test http2 20tps $i"
   echo "GET ${app_url}" | ./go/bin/vegeta attack -duration=300s -http2=true -insecure=true -keepalive=true -rate=20/1s -redirects=1 -workers=2 | tee results.bin | ./go/bin/vegeta report
done

for i in {1..5}
do
   echo "====="
   echo "`date`: Test http2 30tps $i"
   echo "GET ${app_url}" | ./go/bin/vegeta attack -duration=300s -http2=true -insecure=true -keepalive=true -rate=30/1s -redirects=1 -workers=2 | tee results.bin | ./go/bin/vegeta report
done

for i in {1..3}
do
   echo "====="
   echo "`date`: Test http2 50tps $i"
   echo "GET ${app_url}" | ./go/bin/vegeta attack -duration=300s -http2=true -insecure=true -keepalive=true -rate=50/1s -redirects=1 -workers=2 | tee results.bin | ./go/bin/vegeta report
done

• Create docker network (bridge). Both containers to run on bridge network. And link them using --link argument.

Minimal yet complete reproducer code (or URL to code)

public class NettyServer {

  private static final Logger LOGGER = LoggerFactory.getLogger(NettyServer.class);
  private static final String FORCE_JDK_NIO_ENV = "FORCE_NIO";

  static {
    ResourceLeakDetector.setLevel(Level.DISABLED);
  }

  private final int port;

  public NettyServer(int port) {
    this.port = port;
  }

  public void run() throws Exception {
    // Configure the server.
    final boolean forceNio = Boolean.valueOf(System.getenv(FORCE_JDK_NIO_ENV));
    if (Epoll.isAvailable() && !forceNio) {
      doRun(new EpollEventLoopGroup(1), EpollServerSocketChannel.class, IoMultiplexer.EPOLL);
    } else {
      doRun(new NioEventLoopGroup(1), NioServerSocketChannel.class, IoMultiplexer.JDK);
    }
  }

  private void doRun(EventLoopGroup loupGroup, Class<? extends ServerChannel> serverChannelClass,
      IoMultiplexer multiplexer) throws InterruptedException {
    try {
      InetSocketAddress inet = new InetSocketAddress(port);

      ServerBootstrap b = new ServerBootstrap();

      if (multiplexer == IoMultiplexer.EPOLL) {
        b.option(EpollChannelOption.SO_REUSEPORT, true);
      }

      b.option(ChannelOption.SO_BACKLOG, 8192);
      b.option(ChannelOption.SO_REUSEADDR, true);
      b.group(loupGroup).channel(serverChannelClass)
          .childHandler(new ServerChannelInitializer());
      b.childOption(ChannelOption.SO_REUSEADDR, true);
      Channel ch = b.bind(inet).sync().channel();

      LOGGER.info("Httpd started. Listening on: {}", inet.toString());

      ch.closeFuture().sync();
    } finally {
      loupGroup.shutdownGracefully().sync();
    }
  }

  public static void main(String[] args) throws Exception {
    int port;
    if (args.length > 0) {
      port = Integer.parseInt(args[0]);
    } else {
      port = 8443;
    }
    new NettyServer(port).run();
  }
}

public class ServerChannelInitializer extends ChannelInitializer<SocketChannel> {

  private static final EventExecutorGroup group = new DefaultEventExecutorGroup(1);
  private boolean pipelineInIoThread = true;

  public ServerChannelInitializer() {
    super();
  }

  public ServerChannelInitializer(boolean pipelineInIoThread) {
    this();
    this.pipelineInIoThread = pipelineInIoThread;
  }

  @Override
  public void initChannel(SocketChannel ch) throws Exception {
    if (this.pipelineInIoThread) {
      ch.pipeline()
          .addLast("encoder", new HttpResponseEncoder())
          .addLast("decoder", new HttpRequestDecoder(4096, 8192, 8192, false))
          .addLast("handler", new MessageHandler());
    } else {
      ch.pipeline()
          .addLast(group, "encoder", new HttpResponseEncoder())
          .addLast(group, "decoder", new HttpRequestDecoder(4096, 8192, 8192, false))
          .addLast(group, "handler", new MessageHandler());
    }

  }

}

public class MessageHandler extends ChannelInboundHandlerAdapter {

  private static ObjectMapper newMapper() {
    ObjectMapper m = new ObjectMapper();
    m.registerModule(new AfterburnerModule());
    return m;
  }

  private static final String PATH_HEALTH = "/health";
  private HealthCheckHandler healthCheckHandler;

  public MessageHandler() {
    super();
    healthCheckHandler = new HealthCheckHandler();
    healthCheckHandler.setHealthChecker(new AlwaysOkHealthChecker());
  }

  @Override
  public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
    if (msg instanceof HttpRequest) {
      try {
        HttpRequest request = (HttpRequest) msg;
        process(ctx, request);
      } finally {
        ReferenceCountUtil.release(msg);
      }
    } else {
      ctx.fireChannelRead(msg);
    }
  }

  private void process(ChannelHandlerContext ctx, HttpRequest request) throws Exception {
    String uri = request.uri();
    switch (uri) {
      case PATH_HEALTH:
        final ResponseValue responseValue = healthCheckHandler.handle(null);
        ctx.write(makeResponse(responseValue.getResponse().getByteBuf(), APPLICATION_JSON,
            AsciiString.cached(String.valueOf(responseValue.getResponse().length()))),
            ctx.voidPromise());
        return;
    }
    FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, NOT_FOUND,
        Unpooled.EMPTY_BUFFER, false);
    ctx.write(response).addListener(ChannelFutureListener.CLOSE);
  }

  private FullHttpResponse makeResponse(ByteBuf buf, CharSequence contentType,
      CharSequence contentLength) {
    final FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK, buf, false);
    response.headers()
        .set(CONTENT_TYPE, contentType)
        .set(CONTENT_LENGTH, contentLength);
    return response;
  }


  @Override
  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
    ctx.close();
  }

  @Override
  public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
    ctx.flush();
  }

}

Netty version

4.1.25.Final

JVM version (e.g. java -version)

openjdk version "11" 2018-09-25
OpenJDK Runtime Environment 18.9 (build 11+28)
OpenJDK 64-Bit Server VM 18.9 (build 11+28, mixed mode)
-XX:+DisableExplicitGC -XX:InitialHeapSize=16777216 -XX:MaxHeapSize=268435456 -XX:+PrintCommandLineFlags -XX:ReservedCodeCacheSize=251658240 -XX:+SegmentedCodeCache -XX:-UseAdaptiveSizePolicy -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseSerialGC

OS version (e.g. uname -a)

Red Hat Enterprise Linux Server release 7.4 (Maipo)
Linux ad41f683e30e 3.10.0-862.11.6.el7.x86_64 #1 SMP Fri Aug 10 16:55:11 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

Start with the NettyServer, ServerChannelInitializer, and MessageHandler reproducer, then run the provided Vegeta HTTP/2 tests against the /health endpoint in the one-CPU Docker setup. Compare the reported latency percentiles with the EPoll and NIO paths and the pipeline execution options. Done means identifying and addressing the source of the excess latency, with measurements showing the resulting behavior.