`httpx.AsyncClient` has much worse performance than `aiohttp.ClientSession` with concurrent requests

Originally opened by @MarkusSintonen on 2024-06-04 09:30:13 in encode/httpx

There seems to be some performance issues in httpx as it has much worse performance compared to using aiohttp with concurrently running requests. The following benchmark shows how running 20 requests concurrently is over 10x slower with httpx compared to aiohttp. The benchmark has very basic httpx usage for doing multiple GET requests with limited concurrency. The script outputs a figure showing how duration of each GET request has huge variance with httpx.

# 1. start server: python bench.py server
# 2. run client test: python bench.py client

import asyncio
import sys
from typing import Any, Coroutine, Iterator
import aiohttp
import time
import httpx
from aiohttp import web
import matplotlib.pyplot as plt


PORT = 1234
URL = f"http://localhost:{PORT}/req"
RESP = "a" * 2000
REQUESTS = 100
CONCURRENCY = 20


def run_web_server():
    async def handle(_request):
        return web.Response(text=RESP)

    app = web.Application()
    app.add_routes([web.get('/req', handle)])
    web.run_app(app, host="localhost", port=PORT)


def duration(start: float) -> int:
    return int((time.monotonic() - start) * 1000)


async def run_requests(axis: plt.Axes):
    async def gather_limited_concurrency(coros: Iterator[Coroutine[Any, Any, Any]]):
        sem = asyncio.Semaphore(CONCURRENCY)
        async def coro_with_sem(coro):
            async with sem:
                return await coro
        return await asyncio.gather(*(coro_with_sem(c) for c in coros))

    async def httpx_get(session: httpx.AsyncClient, timings: list[int]):
        start = time.monotonic()
        res = await session.request("GET", URL)
        assert len(await res.aread()) == len(RESP)
        assert res.status_code == 200, f"status_code={res.status_code}"
        timings.append(duration(start))

    async def aiohttp_get(session: aiohttp.ClientSession, timings: list[int]):
        start = time.monotonic()
        async with session.request("GET", URL) as res:
            assert len(await res.read()) == len(RESP)
            assert res.status == 200, f"status={res.status}"
        timings.append(duration(start))

    async with httpx.AsyncClient() as session:
        # warmup
        await asyncio.gather(*(httpx_get(session, []) for _ in range(REQUESTS)))

        timings = []
        start = time.monotonic()
        await gather_limited_concurrency((httpx_get(session, timings) for _ in range(REQUESTS)))
        axis.plot([*range(REQUESTS)], timings, label=f"httpx (tot={duration(start)}ms)")

    async with aiohttp.ClientSession() as session:
        # warmup
        await asyncio.gather(*(aiohttp_get(session, []) for _ in range(REQUESTS)))

        timings = []
        start = time.monotonic()
        await gather_limited_concurrency((aiohttp_get(session, timings) for _ in range(REQUESTS)))
        axis.plot([*range(REQUESTS)], timings, label=f"aiohttp (tot={duration(start)}ms)")


def main(mode: str):
    assert mode in {"server", "client"}, f"invalid mode: {mode}"

    if mode == "server":
        run_web_server()
    else:
        fig, ax = plt.subplots()
        asyncio.run(run_requests(ax))
        plt.legend(loc="upper left")
        ax.set_xlabel("# request")
        ax.set_ylabel("[ms]")
        plt.show()

    print("DONE", flush=True)


if __name__ == "__main__":
    assert len(sys.argv) == 2, f"Usage: {sys.argv[0]} server|client"
    main(sys.argv[1])

I found the following issue but seems its not related as the workaround doesnt make a difference here https://github.com/encode/httpx/issues/838#issuecomment-1291224189

Start with the supplied benchmark script and run python bench.py server, then python bench.py client to reproduce the concurrent-request comparison. Compare the httpx.AsyncClient and aiohttp.ClientSession paths and their timing output; done means the cause of the performance gap is identified and the benchmark shows comparable behavior after the fix.