Routing table retains stale entries for short-lived shards

#!/usr/bin/env bash
# Reproduces the 503 "no shards available" bug (commits 92a526b → e1732a7).
# Creates index → ingests → deletes → re-creates → ingests again (should 503 on buggy builds).
set -euo pipefail

IMAGE="${QUICKWIT_IMAGE:-quickwit/quickwit:v0.9.0-rc}"
CONTAINER="qw-repro-503-$$"
URL="http://localhost:7280"
INDEX="test-index-repro"

trap 'echo "Cleaning up..."; docker rm -f "$CONTAINER" >/dev/null 2>&1 || true' EXIT

NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
PAYLOAD='{"timestamp":"'"$NOW"'","message":"msg1"}
{"timestamp":"'"$NOW"'","message":"msg2"}
{"timestamp":"'"$NOW"'","message":"msg3"}'

INDEX_CONFIG='{"version":"0.8","index_id":"'"$INDEX"'","doc_mapping":{"field_mappings":[{"name":"timestamp","type":"datetime","input_formats":["iso8601"],"fast":true},{"name":"message","type":"text"}],"timestamp_field":"timestamp"}}'

# Start Quickwit
echo "Starting Quickwit ($IMAGE)..."
docker run -d --name "$CONTAINER" -p 7280:7280 -e QW_DISABLE_TELEMETRY=true RUST_LOG=debug "$IMAGE" run >/dev/null

echo "Waiting for readiness..."
for i in $(seq 1 60); do
    curl -sf "$URL/health/readyz" >/dev/null 2>&1 && break
    [ "$i" -eq 60 ] && { echo "FAIL: not ready after 60s"; exit 1; }
    sleep 1
done
echo "Ready."

# Round 1: create → ingest → delete
echo "--- Round 1 ---"
curl -sf -X POST "$URL/api/v1/indexes" -H "Content-Type: application/json" -d "$INDEX_CONFIG" >/dev/null
echo "Index created."

curl -sf -X POST "$URL/api/v1/$INDEX/ingest" -H "Content-Type: application/x-ndjson" -d "$PAYLOAD" >/dev/null
echo "Ingest OK."

curl -sf -X DELETE "$URL/api/v1/indexes/$INDEX" >/dev/null
echo "Index deleted."

# Wait
echo "Waiting 10s..."
sleep 10

# Round 2: re-create → ingest (expected to fail with bug)
echo "--- Round 2 ---"
curl -sf -X POST "$URL/api/v1/indexes" -H "Content-Type: application/json" -d "$INDEX_CONFIG" >/dev/null
echo "Index re-created."

# If we wait here, the bug doesn't reproduce (maybe because the shards have time to be marked as failed and removed from the cluster state?).
# sleep 10

echo "Ingesting (round 2)..."
curl -sv -X POST "$URL/api/v1/$INDEX/ingest" -H "Content-Type: application/x-ndjson" -d "$PAYLOAD"