Peer refresh latches a stale node list into a raft conf change that never times out

#!/bin/bash
# Issue: Peer refresh latches a stale node list into a raft conf change that never times out
# Typesense Version: 30.2
# Description:
#   ReplicationState::refresh_nodes() re-reads the --nodes file every 10s and, on the
#   leader, calls braft change_peers() with whatever that file said on THAT tick.
#   braft snapshots the requested configuration in ConfigurationCtx::start() and then
#   rejects every later change_peers() with EBUSY "Doing another configuration change".
#   It never merges the newer target, and NodeImpl::on_caughtup() re-arms the catch-up
#   wait for as long as the joining peer answers RPCs, so nothing bounds how long the
#   stale target stays latched.
#
#   Result: one tick that reads an incomplete nodes file (2 of 3 peers) while a peer is
#   slow to catch up locks the third node out of the cluster. Correcting the nodes file
#   afterwards has no effect: every refresh is refused for the whole catch-up.
#
#   This script builds a 3 node cluster where node 3 is added to the nodes file AFTER a
#   2 peer change_peers is already in flight, then shows node 3 never joins.
#
# Requires: docker, curl, awk

set -e

# On Windows Git Bash, MSYS rewrites absolute paths like /data into
# C:/Program Files/Git/data before handing them to docker.exe, which breaks the
# bind mount and --data-dir so Typesense cannot find its data directory.
# Turning path conversion off keeps those paths literal. It is inert on Linux
# and macOS, so the script stays portable.
export MSYS_NO_PATHCONV=1

# ============================================================================
# CONFIGURATION
# ============================================================================

TYPESENSE_API_KEY=xyz
VERSION=30.2

NET=ts-conflatch-net
N1=ts-conflatch-1          # leader
N2=ts-conflatch-2          # slow joiner, holds the conf change lock
N3=ts-conflatch-3          # added to the nodes file later, gets locked out

P1=18108
P2=18208
P3=18308

H1=http://localhost:${P1}
H3=http://localhost:${P3}

BASE=$(pwd)
D1=${BASE}/ts-data-1
D2=${BASE}/ts-data-2
D3=${BASE}/ts-data-3

NUM_DOCS=400000            # doc COUNT is what makes the joiner slow to load its snapshot
CHUNK=50000
OBSERVE_SECONDS=90         # how long we watch the corrected nodes file being ignored

PEER1=${N1}:8107:8108
PEER2=${N2}:8107:8108
PEER3=${N3}:8107:8108

# ============================================================================
# CLEANUP
# ============================================================================

cleanup() {
  echo ""
  echo "=== Cleanup ==="
  rm -f "${BASE}/.writer_on"
  for c in ${N1} ${N2} ${N3}; do
    docker stop ${c} > /dev/null 2>&1 || true
    docker rm ${c} > /dev/null 2>&1 || true
  done
  docker network rm ${NET} > /dev/null 2>&1 || true
  rm -rf "${D1}" "${D2}" "${D3}" "${BASE}/docs.jsonl" "${BASE}"/docs-chunk-*
  echo "Cleanup complete"
}
trap cleanup EXIT

# ============================================================================
# HELPERS
# ============================================================================

start_node() {
  local name=$1 port=$2 dir=$3
  shift 3
  docker run -d \
    --name ${name} \
    --network ${NET} \
    -p ${port}:8108 \
    -v "${dir}:/data" \
    typesense/typesense:${VERSION} \
    --data-dir /data \
    --api-key=${TYPESENSE_API_KEY} \
    --nodes=/data/nodes \
    --snapshot-interval-seconds=10 \
    --snapshot-max-byte-count-per-rpc=4096 \
    "$@" > /dev/null
}

wait_for_ok() {
  local url=$1 max=${2:-90} count=0
  while [ $count -lt $max ]; do
    if curl -s "${url}/health" 2>/dev/null | grep -q '"ok":true'; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  return 1
}

# After creating a collection, Typesense may queue the operation via Raft
# consensus. The API returns immediately, but the collection may not be
# queryable yet. This function polls until the collection is available.
wait_for_collection() {
  local collection=$1 max_wait=${2:-30} count=0
  while [ $count -lt $max_wait ]; do
    if curl -s -o /dev/null -w "%{http_code}" "${H1}/collections/${collection}" \
      -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" 2>/dev/null | grep -q "200"; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  echo "WARNING: Collection '${collection}' not ready after ${max_wait}s"
  return 1
}

wait_for_log() {
  local container=$1 pattern=$2 max=${3:-120} count=0
  while [ $count -lt $max ]; do
    if docker logs ${container} 2>&1 | grep -q "${pattern}"; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  return 1
}

# ============================================================================
# SETUP: single node leader
# ============================================================================

echo "=== Setup: starting node 1 as a single node cluster ==="

for c in ${N1} ${N2} ${N3}; do
  docker stop ${c} > /dev/null 2>&1 || true
  docker rm ${c} > /dev/null 2>&1 || true
done
docker network rm ${NET} > /dev/null 2>&1 || true
rm -rf "${D1}" "${D2}" "${D3}"

docker network create ${NET} > /dev/null
mkdir -p "${D1}" "${D2}" "${D3}"

# node 1 starts alone, so its raft configuration is exactly {node 1}
printf '%s' "${PEER1}" > "${D1}/nodes"

start_node ${N1} ${P1} "${D1}"

echo "Waiting for node 1 to become leader..."
wait_for_ok ${H1} || { echo "node 1 never became healthy"; docker logs --tail 40 ${N1}; exit 1; }
echo "node 1 is leader, nodes file = $(cat "${D1}/nodes")"

# ============================================================================
# SEED: enough data that a joining node needs a slow snapshot install
# ============================================================================

echo ""
echo "=== Seeding ${NUM_DOCS} documents ==="

curl -s "${H1}/collections" \
  -X POST \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "docs",
    "fields": [
      {"name": "title", "type": "string"},
      {"name": "body", "type": "string"}
    ]
  }' > /dev/null

wait_for_collection "docs"

awk -v n=${NUM_DOCS} 'BEGIN{
  pad="";
  for (i = 0; i < 200; i++) pad = pad "x";
  for (i = 1; i <= n; i++) printf "{\"id\":\"%d\",\"title\":\"doc %d\",\"body\":\"%s\"}\n", i, i, pad;
}' > "${BASE}/docs.jsonl"

split -l ${CHUNK} "${BASE}/docs.jsonl" "${BASE}/docs-chunk-"
for chunk in "${BASE}"/docs-chunk-*; do
  curl -s "${H1}/collections/docs/documents/import?action=create" \
    -X POST \
    -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
    -H "Content-Type: text/plain" \
    --data-binary @- < "${chunk}" > /dev/null
done

echo "Seeded $(curl -s "${H1}/collections/docs" -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" | grep -o '"num_documents":[0-9]*') "

echo "Forcing a raft snapshot on node 1 (truncates the log, so a joiner must install a snapshot)..."
curl -s -X POST "${H1}/operations/snapshot?snapshot_path=/data/backup" \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" > /dev/null
wait_for_log ${N1} "snapshot_save_done" 120 || { echo "no snapshot was taken"; docker logs --tail 30 ${N1}; exit 1; }
echo "Snapshot taken."

# ============================================================================
# TRIGGER: one tick reads an INCOMPLETE nodes file while node 2 is joining
# ============================================================================

echo ""
echo "=== Trigger: nodes file lists only 2 of the 3 peers on this tick ==="

# A joining node boots with the leader as its only known peer, so it is not in its
# own raft configuration and cannot campaign. That is the state a replacement node
# is really in after it installs a snapshot whose embedded conf is the leader only,
# and it keeps the joiner from bumping the term and disrupting the leader.
printf '%s' "${PEER1}" > "${D2}/nodes"

# node 2 loads its snapshot one document at a time, so the catch-up is slow.
# This is the same state a cold loading replacement node is in: it answers raft
# RPCs the whole time, which is exactly what keeps braft re-arming the wait.
start_node ${N2} ${P2} "${D2}" --num-documents-parallel-load=1

# Wait until node 2 has actually parsed that file, otherwise a later rewrite races
# its startup and it comes up able to campaign.
wait_for_log ${N2} "Nodes configuration:" 90 || { echo "node 2 never read its nodes file"; exit 1; }

# THE BUG TRIGGER: on this tick the leader's nodes file names only 2 of the 3 peers.
printf '%s' "${PEER1},${PEER2}" > "${D1}/nodes"
echo "nodes file on node 1 is now: $(cat "${D1}/nodes")"

echo "Waiting for the leader to start the 2 peer conf change..."
wait_for_log ${N1} "begin caughtup" 120 || { echo "leader never started a conf change"; docker logs --tail 40 ${N1}; exit 1; }

LATCHED=$(docker logs ${N1} 2>&1 | grep "begin caughtup" | tail -1)
echo "Latched conf change: ${LATCHED}"

# ============================================================================
# CORRECT THE NODES FILE, then watch it be ignored
# ============================================================================

echo ""
echo "=== Correcting the nodes file to all 3 peers and starting node 3 ==="

MARK=$(docker logs ${N1} 2>&1 | wc -l)

# Same as node 2: node 3 boots knowing only the leader, so it cannot campaign.
printf '%s' "${PEER1}" > "${D3}/nodes"
start_node ${N3} ${P3} "${D3}"
wait_for_log ${N3} "Nodes configuration:" 90 || { echo "node 3 never read its nodes file"; exit 1; }

# The nodes file is now correct and complete on every node, and every node
# re-reads it every 10s.
printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D1}/nodes"
printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D2}/nodes"
printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D3}/nodes"

echo "nodes file on node 1 is now: $(cat "${D1}/nodes")"

echo "Observing for up to ${OBSERVE_SECONDS}s with the correct 3 peer nodes file in place..."

# Measure how long the corrected nodes file stays un-appliable. Stop early if the
# leader ever manages to start a new conf change, because that is the lock releasing.
STARTED_AT=$(date +%s)
LOCKOUT=0
SAMPLES=0
N3_UNHEALTHY_SAMPLES=0
while [ ${LOCKOUT} -lt ${OBSERVE_SECONDS} ]; do
  sleep 5
  LOCKOUT=$(( $(date +%s) - STARTED_AT ))
  SAMPLES=$((SAMPLES + 1))
  NEW_CONF_CHANGES=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "begin caughtup" || true)
  if [ "$(curl -s --max-time 3 -o /dev/null -w '%{http_code}' "${H3}/health" 2>/dev/null || echo 000)" != "200" ]; then
    N3_UNHEALTHY_SAMPLES=$((N3_UNHEALTHY_SAMPLES + 1))
  fi
  if [ "${NEW_CONF_CHANGES}" -gt 0 ]; then
    echo "Lock released after ${LOCKOUT}s: the leader started a new conf change."
    break
  fi
done

# ============================================================================
# CAPTURE
# ============================================================================

echo ""
echo "=== Leader log since the nodes file was corrected ==="
docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) \
  | grep -E "Refusing concurrent|Doing another configuration change" \
  | tail -6 || true
echo "--- and the wait that is holding the lock, re-armed every 5s with no deadline:"
docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep "to catch up" | tail -3 || true

REFUSALS=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "Doing another configuration change" || true)

echo ""
echo "=== Node 3 (the peer added to the nodes file after the conf change started) ==="
docker logs ${N3} 2>&1 | grep -E "can't do pre_vote as it is not in|Multi-node with no leader" | tail -3 || true

N3_HEALTH=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" "${H3}/health" 2>/dev/null || true)
[ -z "${N3_HEALTH}" ] && N3_HEALTH="000"
# node 3 parsed a MULTI node file (this branch is only reachable when the parsed peer
# count is not 1, raft_server.cpp:783-788) and still has no leader, so it is outside
# the raft configuration despite being listed in every nodes file on every node.
N3_EXCLUDED=$(docker logs ${N3} 2>&1 | grep -c "Multi-node with no leader" || true)

echo ""
echo "=== Summary ==="
echo "nodes file on the leader        : $(cat "${D1}/nodes")"
echo "latched conf change target      : $(echo "${LATCHED}" | sed 's/.*change_peers from/change_peers from/')"
echo "peer refreshes refused (EBUSY)  : ${REFUSALS}"
echo "new conf changes started        : ${NEW_CONF_CHANGES}"
echo "node 3 locked out for           : ${LOCKOUT}s"
echo "node 3 /health while locked out : ${N3_UNHEALTHY_SAMPLES} of ${SAMPLES} samples not 200"
echo "node 3 /health now              : ${N3_HEALTH}"
echo "node 3 'not in configuration'   : ${N3_EXCLUDED} log lines"

# ============================================================================
# VERDICT
# ============================================================================

echo ""
if [ "${REFUSALS}" -ge 3 ] && [ "${LOCKOUT}" -ge 30 ] \
   && [ "${N3_UNHEALTHY_SAMPLES}" -ge $((SAMPLES - 1)) ] && [ "${N3_EXCLUDED}" -ge 3 ]; then
  echo "BUG REPRODUCED"
  echo "The nodes file listed all 3 peers and was re-read every 10s, but for ${LOCKOUT}s the"
  echo "leader kept applying the 2 peer target it captured on one earlier tick. Every"
  echo "refresh in that window was refused with 'Doing another configuration change', so"
  echo "node 3 stayed out of the cluster and served 503 the whole time."
  echo ""
  echo "The window lasted exactly as long as node 2's catch-up, and nothing bounds that:"
  echo "braft re-arms the wait every 5s for as long as the joining peer answers RPCs"
  echo "(NodeImpl::on_caughtup, node.cpp:748-770), and ConfigurationCtx has no deadline."
  echo "A peer that takes an hour to load holds node 3 out for that hour."
  exit 0
else
  echo "Reproducer did not hit the expected state"
  echo "(expected: refusals >= 3, lockout >= 30s, node 3 never healthy, node 3 excluded >= 3)"
  exit 1
fi

#!/bin/bash
# Issue: the latched raft conf change has no deadline - the refusal loop lasts as
#        long as the joining peer's catch-up, and nothing bounds that.
# Typesense Version: 30.2
#
# Description:
#   reproduce.sh already shows the latch: one tick reads a 2 peer nodes file, braft
#   snapshots that target in ConfigurationCtx::start(), and every later change_peers()
#   is refused with EBUSY "Doing another configuration change".
#
#   But on a fast host the joiner catches up in about a minute, so the refusal loop
#   ends on its own and looks self healing. It is not. NodeImpl::on_caughtup()
#   (braft node.cpp:748-770) re-arms wait_caughtup every election_timeout_ms for as
#   long as the joining peer ANSWERS RPCs, and ConfigurationCtx carries no timer.
#   "Alive but slow" therefore holds the lock with no upper bound.
#
#   This script pins that down by starving the joiner of CPU instead of changing
#   anything about the bug: same trigger, same code path, a joiner that is healthy
#   and answering raft RPCs the whole time but cannot finish loading. The refusal
#   loop then runs for the entire observation window with zero releases, and the
#   locked out node serves 503 for all of it.
#
#   The knob is the joiner's speed, nothing else. That is the point: the window is
#   a function of how slow the replacement is, so on a real cluster with a real
#   dataset it is hours, and no operator action shortens it.
#
# Requires: docker, curl, awk

set -e

export MSYS_NO_PATHCONV=1

# ============================================================================
# CONFIGURATION
# ============================================================================

TYPESENSE_API_KEY=xyz
VERSION=30.2

NET=ts-etern-net
N1=ts-etern-1              # leader
N2=ts-etern-2              # CPU starved joiner, holds the conf change lock
N3=ts-etern-3              # added to the nodes file later, gets locked out

P1=19108
P2=19208
P3=19308

H1=http://localhost:${P1}
H2=http://localhost:${P2}
H3=http://localhost:${P3}

BASE=$(pwd)
D1=${BASE}/ts-etern-data-1
D2=${BASE}/ts-etern-data-2
D3=${BASE}/ts-etern-data-3

NUM_DOCS=600000
CHUNK=50000

# The joiner gets a fraction of one core. reproduce.sh on an unthrottled host
# released the lock in 59s; this is the same run with the joiner slowed down.
JOINER_CPUS=0.2

# How long we watch the corrected 3 peer nodes file be ignored.
OBSERVE_SECONDS=400
# The claim under test: the latch survives at least this long. Anything at or
# past this is already 5x the unthrottled run and the only thing that changed
# is how fast the joiner loads.
MIN_LOCKOUT=300

PEER1=${N1}:8107:8108
PEER2=${N2}:8107:8108
PEER3=${N3}:8107:8108

# ============================================================================
# CLEANUP
# ============================================================================

cleanup() {
  echo ""
  echo "=== Cleanup ==="
  for c in ${N1} ${N2} ${N3}; do
    docker stop ${c} > /dev/null 2>&1 || true
    docker rm ${c} > /dev/null 2>&1 || true
  done
  docker network rm ${NET} > /dev/null 2>&1 || true
  rm -rf "${D1}" "${D2}" "${D3}" "${BASE}/etern-docs.jsonl" "${BASE}"/etern-chunk-*
  echo "Cleanup complete"
}
trap cleanup EXIT

# ============================================================================
# HELPERS
# ============================================================================

# $1 name, $2 host port, $3 data dir, $4 docker-level flags (may be empty),
# rest: typesense flags
start_node() {
  local name=$1 port=$2 dir=$3 dockerflags=$4
  shift 4
  docker run -d \
    --name ${name} \
    --network ${NET} \
    -p ${port}:8108 \
    -v "${dir}:/data" \
    ${dockerflags} \
    typesense/typesense:${VERSION} \
    --data-dir /data \
    --api-key=${TYPESENSE_API_KEY} \
    --nodes=/data/nodes \
    --snapshot-interval-seconds=10 \
    --snapshot-max-byte-count-per-rpc=4096 \
    "$@" > /dev/null
}

wait_for_ok() {
  local url=$1 max=${2:-90} count=0
  while [ $count -lt $max ]; do
    if curl -s "${url}/health" 2>/dev/null | grep -q '"ok":true'; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  return 1
}

wait_for_collection() {
  local collection=$1 max_wait=${2:-30} count=0
  while [ $count -lt $max_wait ]; do
    if curl -s -o /dev/null -w "%{http_code}" "${H1}/collections/${collection}" \
      -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" 2>/dev/null | grep -q "200"; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  echo "WARNING: Collection '${collection}' not ready after ${max_wait}s"
  return 1
}

wait_for_log() {
  local container=$1 pattern=$2 max=${3:-120} count=0
  while [ $count -lt $max ]; do
    if docker logs ${container} 2>&1 | grep -q "${pattern}"; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  return 1
}

# ============================================================================
# SETUP
# ============================================================================

echo "=== Setup: starting node 1 as a single node cluster ==="

for c in ${N1} ${N2} ${N3}; do
  docker stop ${c} > /dev/null 2>&1 || true
  docker rm ${c} > /dev/null 2>&1 || true
done
docker network rm ${NET} > /dev/null 2>&1 || true
rm -rf "${D1}" "${D2}" "${D3}"

docker network create ${NET} > /dev/null
mkdir -p "${D1}" "${D2}" "${D3}"

printf '%s' "${PEER1}" > "${D1}/nodes"
start_node ${N1} ${P1} "${D1}" ""

echo "Waiting for node 1 to become leader..."
wait_for_ok ${H1} || { echo "node 1 never became healthy"; docker logs --tail 40 ${N1}; exit 1; }
echo "node 1 is leader, nodes file = $(cat "${D1}/nodes")"

# ============================================================================
# SEED
# ============================================================================

echo ""
echo "=== Seeding ${NUM_DOCS} documents ==="

curl -s "${H1}/collections" \
  -X POST \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "docs",
    "fields": [
      {"name": "title", "type": "string"},
      {"name": "body", "type": "string"}
    ]
  }' > /dev/null

wait_for_collection "docs"

awk -v n=${NUM_DOCS} 'BEGIN{
  pad="";
  for (i = 0; i < 200; i++) pad = pad "x";
  for (i = 1; i <= n; i++) printf "{\"id\":\"%d\",\"title\":\"doc %d\",\"body\":\"%s\"}\n", i, i, pad;
}' > "${BASE}/etern-docs.jsonl"

split -l ${CHUNK} "${BASE}/etern-docs.jsonl" "${BASE}/etern-chunk-"
for chunk in "${BASE}"/etern-chunk-*; do
  curl -s "${H1}/collections/docs/documents/import?action=create" \
    -X POST \
    -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
    -H "Content-Type: text/plain" \
    --data-binary @- < "${chunk}" > /dev/null
done

echo "Seeded $(curl -s "${H1}/collections/docs" -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" | grep -o '"num_documents":[0-9]*')"

echo "Forcing a raft snapshot on node 1 so a joiner must install one..."
curl -s -X POST "${H1}/operations/snapshot?snapshot_path=/data/backup" \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" > /dev/null
wait_for_log ${N1} "snapshot_save_done" 180 || { echo "no snapshot was taken"; docker logs --tail 30 ${N1}; exit 1; }
echo "Snapshot taken."

# ============================================================================
# TRIGGER
# ============================================================================

echo ""
echo "=== Trigger: nodes file lists only 2 of the 3 peers on this tick ==="

# A joining node boots with the leader as its only known peer, so it cannot campaign,
# bump the term and force a step_down (which would call _conf_ctx.reset() and mask
# the bug). That is also the real state of a replacement whose installed snapshot
# carries a leader-only configuration.
printf '%s' "${PEER1}" > "${D2}/nodes"

# The ONLY difference from reproduce.sh: this joiner is CPU starved, so its load
# takes minutes instead of seconds. It stays alive and answers raft RPCs throughout,
# which is exactly what keeps braft re-arming the catch-up wait.
start_node ${N2} ${P2} "${D2}" "--cpus=${JOINER_CPUS}" --num-documents-parallel-load=1

wait_for_log ${N2} "Nodes configuration:" 180 || { echo "node 2 never read its nodes file"; exit 1; }

printf '%s' "${PEER1},${PEER2}" > "${D1}/nodes"
echo "nodes file on node 1 is now: $(cat "${D1}/nodes")"

echo "Waiting for the leader to start the 2 peer conf change..."
wait_for_log ${N1} "begin caughtup" 180 || { echo "leader never started a conf change"; docker logs --tail 40 ${N1}; exit 1; }

LATCHED=$(docker logs ${N1} 2>&1 | grep "begin caughtup" | tail -1)
echo "Latched conf change: ${LATCHED}"

# ============================================================================
# CORRECT THE NODES FILE, then watch it be ignored for the whole window
# ============================================================================

echo ""
echo "=== Correcting the nodes file to all 3 peers and starting node 3 ==="

MARK=$(docker logs ${N1} 2>&1 | wc -l)

printf '%s' "${PEER1}" > "${D3}/nodes"
start_node ${N3} ${P3} "${D3}" ""
wait_for_log ${N3} "Nodes configuration:" 180 || { echo "node 3 never read its nodes file"; exit 1; }

printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D1}/nodes"
printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D2}/nodes"
printf '%s' "${PEER1},${PEER2},${PEER3}" > "${D3}/nodes"

echo "nodes file on every node is now: $(cat "${D1}/nodes")"
echo ""
echo "Observing for up to ${OBSERVE_SECONDS}s. Each line is one 30s sample."
printf '%-9s %-10s %-10s %-9s %-9s\n' "elapsed" "refusals" "released" "n3_health" "n2_health"

STARTED_AT=$(date +%s)
LOCKOUT=0
SAMPLES=0
N3_UNHEALTHY_SAMPLES=0
N2_ALIVE_SAMPLES=0
NEW_CONF_CHANGES=0
RELEASED_AT=""

while [ ${LOCKOUT} -lt ${OBSERVE_SECONDS} ]; do
  sleep 30
  LOCKOUT=$(( $(date +%s) - STARTED_AT ))
  SAMPLES=$((SAMPLES + 1))

  NEW_CONF_CHANGES=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "begin caughtup" || true)
  REFUSALS=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "Doing another configuration change" || true)

  N3_CODE=$(curl -s --max-time 3 -o /dev/null -w '%{http_code}' "${H3}/health" 2>/dev/null || echo 000)
  [ "${N3_CODE}" != "200" ] && N3_UNHEALTHY_SAMPLES=$((N3_UNHEALTHY_SAMPLES + 1))

  # node 2 answering ANY HTTP at all (even 503) means the process is alive; the
  # leader re-arming its wait is the raft-level proof it is answering RPCs too.
  N2_CODE=$(curl -s --max-time 5 -o /dev/null -w '%{http_code}' "${H2}/health" 2>/dev/null || echo 000)
  [ "${N2_CODE}" != "000" ] && N2_ALIVE_SAMPLES=$((N2_ALIVE_SAMPLES + 1))

  printf '%-9s %-10s %-10s %-9s %-9s\n' "${LOCKOUT}s" "${REFUSALS}" "${NEW_CONF_CHANGES}" "${N3_CODE}" "${N2_CODE}"

  if [ "${NEW_CONF_CHANGES}" -gt 0 ]; then
    RELEASED_AT=${LOCKOUT}
    echo "Lock released after ${LOCKOUT}s: the leader started a new conf change."
    break
  fi
done

# ============================================================================
# CAPTURE
# ============================================================================

REFUSALS=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "Doing another configuration change" || true)
WAIT_REARMS=$(docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep -c "to catch up" || true)

echo ""
echo "=== Leader: first and last refusal in the window ==="
docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep "Doing another configuration change" | head -1 || true
docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep "Doing another configuration change" | tail -1 || true

echo ""
echo "=== The wait holding the lock, re-armed with no deadline (last 3) ==="
docker logs ${N1} 2>&1 | tail -n +$((MARK + 1)) | grep "to catch up" | tail -3 || true

echo ""
echo "=== Node 2 is alive and still loading, it never went away ==="
docker logs ${N2} 2>&1 | grep -E "Loading collection|Finished loading|Typesense has started" | tail -3 || true

echo ""
echo "=== Node 3, listed in every nodes file, still outside the configuration ==="
docker logs ${N3} 2>&1 | grep -E "can't do pre_vote as it is not in|Multi-node with no leader" | tail -3 || true

N3_HEALTH=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" "${H3}/health" 2>/dev/null || true)
[ -z "${N3_HEALTH}" ] && N3_HEALTH="000"
N3_EXCLUDED=$(docker logs ${N3} 2>&1 | grep -c "Multi-node with no leader" || true)

echo ""
echo "=== Summary ==="
echo "joiner CPU allotment            : ${JOINER_CPUS} of a core"
echo "nodes file on the leader        : $(cat "${D1}/nodes")"
echo "latched conf change target      : $(echo "${LATCHED}" | sed 's/.*change_peers from/change_peers from/')"
echo "peer refreshes refused (EBUSY)  : ${REFUSALS}"
echo "catch-up wait re-arms           : ${WAIT_REARMS}"
echo "new conf changes started        : ${NEW_CONF_CHANGES}"
echo "latch held for                  : ${LOCKOUT}s${RELEASED_AT:+ (released at ${RELEASED_AT}s)}"
echo "node 3 /health while latched    : ${N3_UNHEALTHY_SAMPLES} of ${SAMPLES} samples not 200"
echo "node 3 /health now              : ${N3_HEALTH}"
echo "node 3 'not in configuration'   : ${N3_EXCLUDED} log lines"
echo "node 2 answering HTTP           : ${N2_ALIVE_SAMPLES} of ${SAMPLES} samples"

# ============================================================================
# VERDICT
# ============================================================================

echo ""
if [ "${LOCKOUT}" -ge ${MIN_LOCKOUT} ] && [ "${REFUSALS}" -ge 25 ] \
   && [ "${N3_UNHEALTHY_SAMPLES}" -ge $((SAMPLES - 1)) ] && [ "${N3_EXCLUDED}" -ge 3 ]; then
  echo "BUG REPRODUCED - latch held ${LOCKOUT}s with no deadline"
  echo ""
  echo "Nothing about the bug changed between this run and reproduce.sh. The only"
  echo "difference is that the joining peer got ${JOINER_CPUS} of a core, so its load takes"
  echo "minutes instead of seconds - and the refusal loop lasted exactly that long."
  echo "It ran for ${LOCKOUT}s with ${REFUSALS} refusals and ${NEW_CONF_CHANGES} new conf changes started,"
  echo "while node 2 stayed alive and the leader re-armed its catch-up wait ${WAIT_REARMS} times."
  echo ""
  echo "That is the unbounded part: the window is a pure function of how slow the"
  echo "replacement is. On a production dataset that is hours, and rewriting the"
  echo "nodes file does nothing for any of it."
  exit 0
else
  echo "Reproducer did not hit the expected state"
  echo "(expected: lockout >= ${MIN_LOCKOUT}s, refusals >= 25, node 3 never healthy, node 3 excluded >= 3)"
  exit 1
fi