fatal error: concurrent map iteration and map write

#### Overview of the Issue

Consul agent running in server mode in a 3 node cluster panics and then crashed. This only happens on the node that is the leader. When this happens cluster leadership is transfered to another node. After some time (could be in a couple of hours or even days) this leader also crashes, leadership transfers to another node and the cycle continues.

---

#### Reproduction Steps

It is not known how to reproduce this or what exactly triggers this. It happens randomly in an otherwise normally functioning cluster.

### Consul info for both Client and Server

Client info

```
agent:
check_monitors = 0
check_ttls = 0
checks = 3
services = 2
build:
prerelease =
revision = 98cb473+
version = 1.18.1
version_metadata =
consul:
acl = disabled
known_servers = 5
server = false
runtime:
arch = amd64
cpu_count = 2
goroutines = 103
max_procs = 2
os = linux
version = go1.22.3
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 806
failed = 0
health_score = 0
intent_queue = 0
left = 5
member_time = 2369641
members = 146
query_queue = 0
query_time = 1646
```

```
{
"datacenter": "dc1",
"client_addr": "127.0.0.1",
"bind_addr": "{{ GetPrivateIP }}",
"ports": {
"grpc": 8502
},
"data_dir": "/consul/data",
"pid_file": "/var/run/consul/consul.pid",
"retry_join": ["provider=aws service=ecs tag_key=role tag_value=consul"],
"limits": {
"http_max_conns_per_client": 500
},
"encrypt": "",
"log_level": "INFO",
"leave_on_terminate": true,
"advertise_reconnect_timeout": "1h",
"cache": {
"entry_fetch_rate": 1
},
"disable_update_check": true,
"telemetry": {
"dogstatsd_addr": "127.0.0.1:8125",
"dogstatsd_tags": [
"service:servicename-service"
]
}
}

{
"service": {
"name": "servicename-service-proxy",
"id": "servicename-service-proxy",
"kind": "connect-proxy",
"proxy": {
"upstreams":[{"destination_name": "egress-proxy","local_bind_address": "127.0.0.1","local_bind_port": 13128,"config":{"connect_timeout_ms":5000}}],
"destination_service_name": "servicename-service",
"destination_service_id": "servicename-service",
"local_service_port": 7843,
"config": {
"local_request_timeout_ms": 300000,
"local_connect_timeout_ms": 5000,
"local_idle_timeout_ms": 60000,
"envoy_tracing_json": "{\"http\":{\"name\":\"envoy.tracers.datadog\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.config.trace.v3.DatadogConfig\",\"collector_cluster\":\"datadog_agent\",\"service_name\":\"servicename-service\"}}}"
}
},
"port": 41641
}
}

{
"check": {
"id": "servicename-service-proxy-healthcheck",
"name": "servicename-service-proxy-healthcheck",
"service_id": "servicename-service-proxy",
"TCP": "127.0.0.1:41641",
"interval": "10s",
"timeout": "5s",
"success_before_passing": 1,
"failures_before_critical": 0
}
}

{
"service": {
"name": "servicename-service",
"id": "servicename-service",
"port": 41141,
"tags": [
"fe_haproxy_http_app","haproxy_http_app","haproxy_httpchk=GET /health/"
]
}
}

{
"check": {
"id": "servicename-service-alias-healthcheck",
"name": "servicename-service-alias-healthcheck",
"service_id": "servicename-service-proxy",
"alias_service": "servicename-service"
}
}

{
"check": {
"id": "servicename-service-healthcheck",
"name": "servicename-service-healthcheck",
"service_id": "servicename-service",
"http": "https://servicename-service.service.consul:41141/health/",
"interval": "5s",
"timeout": "5s",
"success_before_passing": 1,
"failures_before_critical": 0,
"tls_skip_verify": true
}
}
```

Server info

```
agent:
check_monitors = 0
check_ttls = 0
checks = 0
services = 0
build:
prerelease =
revision = 98cb473+
version = 1.18.1
version_metadata =
consul:
acl = disabled
bootstrap = false
known_datacenters = 1
leader = false
leader_addr = 10.140.32.151:8300
server = true
raft:
applied_index = 54165746
commit_index = 54165746
fsm_pending = 0
last_contact = 46.544497ms
last_log_index = 54165747
last_log_term = 19019
last_snapshot_index = 54163954
last_snapshot_term = 19019
latest_configuration = [removed]
latest_configuration_index = 0
num_peers = 2
protocol_version = 3
protocol_version_max = 3
protocol_version_min = 0
snapshot_version_max = 1
snapshot_version_min = 0
state = Follower
term = 19019
runtime:
arch = amd64
cpu_count = 4
goroutines = 1823
max_procs = 4
os = linux
version = go1.22.3
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 806
failed = 0
health_score = 0
intent_queue = 0
left = 7
member_time = 2370742
members = 149
query_queue = 0
query_time = 1646
serf_wan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 1
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 40498
members = 6
query_queue = 0
query_time = 1
```

```
{
"server": true,
"datacenter": "dc1",
"node_name": "consul-server-0",
"bootstrap_expect": 3,
"ui_config": {
"enabled" : true
},
"data_dir": "/consul/data",
"pid_file": "/var/run/consul/consul.pid",
"retry_join": ["provider=aws service=ec2 tag_key=role tag_value=consul",],
"addresses": {
"http": "0.0.0.0",
"https": "0.0.0.0"
},
"bind_addr": "{{ GetPrivateIP }}",
"client_addr": "0.0.0.0",
"ports": {
"http": 8500,
"https": 8501
},
"tls": {
"defaults": {
"key_file": "/consul/certs/consul-server-0.key",
"cert_file": "/consul/certs/consul-server-0.pem",
"ca_file": "/consul/certs/ca.pem"
}
},
"connect": {
"enabled": true
},
"encrypt": "",
"config_entries": {
"bootstrap": [
{
"Kind": "proxy-defaults",
"Name": "global",
"Config": {
"protocol": "http"
}
}
]
},
"domain": "consul",
"leave_on_terminate": true,
"disable_update_check": true,
"log_level": "debug",
"telemetry": {
"enable_host_metrics": true,
"dogstatsd_addr": "127.0.0.1:8125",
"dogstatsd_tags": [
"service:consul"
]
}
}
```

### Operating system and Environment details

The environment has Consul servers running on EC2 instances and ECS tasks running on ECS Fargate. There are about 140 consul clients running
### Log Fragments
Crash log is attached: [crash.log.txt](https://github.com/user-attachments/files/15759925/crash.log.txt)

Start with the attached crash.log.txt and the reported Go runtime panic, then compare the supplied leader and follower Consul server information. The issue does not identify a source file, entry point, reproduction, or failing test. Done means isolating the concurrent map access, establishing a reliable validation path, and confirming that the leader no longer crashes under the relevant cluster workload.

Written by the indexing model from the issue text.