Envoy Listener Not Created via Consul Connect xDS API and Persistent gRPC Stream Warnings

While attempting to use Consul Connect with Envoy as a sidecar proxy, the expected Envoy listener is not created through Consul's xDS API. Additionally, there are persistent warning messages in both Consul and Envoy logs:

**Consul Warning**:` [WARN] agent.grpc.external: failed to determine which rate limit to apply to RPC: rpc=/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources.`

**Envoy Warning**: `[warning][config] [./source/extensions/config_subscription/grpc/grpc_stream.h:152] StreamAggregatedResources gRPC config stream to xds_cluster closed: 2, not implemented.`

These warnings indicate a failure in the gRPC communication between Envoy and Consul, which prevents the proper creation and management of Envoy listeners configured through Consul Connect.

**Expected Behavior**:
The Envoy sidecar should properly receive configuration updates from Consul via the xDS API and create the necessary listeners.
Consul should not display warnings about rate limiting or gRPC configuration for the xDS API if the setup is correct.

**Actual Behavior**:
Envoy does not create the expected listener that should be configured through Consul’s xDS API.
Consul logs repeated warnings about failing to determine which rate limit to apply to RPC.
Envoy logs warnings about the gRPC stream to xds_cluster being closed with a "not implemented" status.

**Steps to Reproduce**:

1. Set up Consul with the below configuration
3. run envoy as docker container via nomad with the below configuration
4. register a service to consul that listens to port 8080, with an http listener that redirect to https (the listener config doesn't really matter)
7. Start the Consul agent and Envoy sidecar - consul connect proxy -service=envoy -sidecar-for tomcat
8. Observe the Consul and Envoy logs for the described warnings.

**Environment Details:**
Consul Version: 1.17.0-1 (RPM)
Envoy Version: 1.27-latest (Docker image)
Nomad Version: 1.6.1 (RPM)
Operating System: [CENTOS7]

consul config
```
{
"addresses": {
"dns": "127.0.0.1",
"grpc": "127.0.0.1",
"http": "127.0.0.1",
"https": "127.0.0.1"
},
"connect": {
"enabled": true
},
"limits": {
"request_limits": {
"mode": "enforcing",
"read_rate": 500,
"write_rate": 200
},
"rpc_rate": 1000
},
"advertise_addr": "ip",
"advertise_addr_wan": "ip",
"auto_encrypt": {},
"bind_addr": "ip",
"bootstrap": false,
"bootstrap_expect": 1,
"tls": {
"defaults": {
"ca_file": "ca.crt",
"cert_file": "certificate.crt",
"key_file": "private.key",
"tls_cipher_suites": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"tls_min_version": "TLSv1_2",
"verify_incoming": false,
"verify_outgoing": true
},
"https": {
"verify_incoming": false
},
"internal_rpc": {
"verify_incoming": false,
"verify_server_hostname": false
},
"grpc": {
"tls_min_version": "TLSv1_2",
"verify_incoming": true
}
},
"client_addr": "127.0.0.1",
"data_dir": "/usr/local/lib/ctera/work/microservices/var/consul",
"datacenter": "dc1",
"disable_update_check": true,
"domain": "ctera",
"enable_local_script_checks": false,
"enable_script_checks": true,
"encrypt": "",
"encrypt_verify_incoming": true,
"encrypt_verify_outgoing": true,
"log_file": "consul.log",
"log_level": "INFO",
"log_rotate_bytes": 1073741824,
"log_rotate_duration": "24h",
"log_rotate_max_files": 20,
"node_name": "consul-server-6",
"ports": {
"dns": 8600,
"grpc_tls": 8502,
"http": 8500,
"https": -1,
"serf_lan": 8301,
"serf_wan": -1,
"server": 8300
},
"raft_protocol": 3,
"retry_interval": "30s",
"retry_interval_wan": "30s",
"retry_join": [
"192.168.14.88"
],
"xds": {},
"retry_max": 0,
"retry_max_wan": 0,
"server": true,
"translate_wan_addrs": false,
"ui_config": {
"enabled": true
}
}

```

service registration config
```
{
"service": {
"name": "service",
"port": 8080,
"connect": {
"sidecar_service": {
"proxy": {
"config": {
"protocol": "grpc_tls",
"listeners": [
{
"name": "http_listener",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 80
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "ingress_http",
"route_config": {
"name": "local_route",
"virtual_hosts": [
{
"name": "service_redirect",
"domains": ["*"],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "portal-cluster"
},
"response_headers_to_remove": [
"x-envoy-upstream-service-time"
]
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"server_header_transformation": "PASS_THROUGH"
}
}
],
"transport_socket": {
"name": "envoy.transport_sockets.tls",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"common_tls_context": {
"tls_certificate_sds_secret_configs": {
"name": "tls_sds",
"sds_config": {
"path": "/etc/envoy/xds/server-sds-tls-certificate.json",
"resource_api_version": "V3"
}
},
"validation_context_sds_secret_config": {
"name": "validation_context_sds",
"sds_config": {
"path": "/etc/envoy/xds/server-sds-validation-context.json",
"resource_api_version": "V3"
}
}
},
"require_client_certificate": true
}
}
}
]
}
]
}
}
}
}
}
}

```

envoy config:

```
node:
id: main
cluster: Multi_Service

dns_resolution_config:
resolvers:
- socket_address:
address: 127.0.0.1
port_value: 8600

dynamic_resources:
ads_config:
api_type: GRPC
transport_api_version: V3
grpc_services:
- envoy_grpc:
cluster_name: xds_cluster
cds_config:
ads: {}
lds_config:
ads: {}

static_resources:
listeners:
- {
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 443
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"codec_type": "auto",
"http_filters": [
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"route_config": {
"name": "local_route",
"virtual_hosts": [
{
"domains": [
"*"
],
"name": "backend",
"routes": [
{
"match": {
"prefix": "/",
},
"route": {
"cluster": "service-cluster",
"prefix_rewrite": "/",
"timeout": "0s",
}
}
]
}
]
},
"stat_prefix": "multi_services",
"server_header_transformation": "PASS_THROUGH"
}
}
],
"transport_socket": {
"name": "envoy.transport_sockets.tls",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"common_tls_context": {
"tls_certificate_sds_secret_configs": {
"name": "tls_sds",
"sds_config": {
"path": "/etc/envoy/xds/server-sds-tls-certificate.json",
"resource_api_version": "V3"
}
},
"validation_context_sds_secret_config": {
"name": "validation_context_sds",
"sds_config": {
"path": "/etc/envoy/xds/server-sds-validation-context.json",
"resource_api_version": "V3"
}
}
},
"require_client_certificate": false
}
}
}
],
"listener_filters": [
{
"name": "envoy.filters.listener.tls_inspector",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector"
}
}
],
"name": "front-multi_service-listener"
}
clusters:

- {
"connect_timeout": "5s",
"lb_policy": "round_robin",
"load_assignment": {
"cluster_name": "service-cluster",
"endpoints": [
{
"lb_endpoints": [
{
"endpoint": {
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 8080
}
}
}
}
]
}
]
},
"name": "service-cluster",
"type": "static"
}

- {
"connect_timeout": "1s",
"lb_policy": "round_robin",
"load_assignment": {
"cluster_name": "xds_cluster",
"endpoints": [
{
"lb_endpoints": [
{
"endpoint": {
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
}
}
]
}
]
},
"http2_protocol_options": {},
"name": "xds_cluster",
"transport_socket": {
"name": "envoy.transport_sockets.tls",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"common_tls_context": {
"tls_certificate_sds_secret_configs": {
"name": "tls_sds",
"sds_config": {
"path": "/etc/envoy/xds/sds-tls-certificate.json",
"resource_api_version": "V3"
}
},
"validation_context_sds_secret_config": {
"name": "validation_context_sds",
"sds_config": {
"path": "/etc/envoy/xds/sds-validation-context.json",
"resource_api_version": "V3"
}
}
}
}
},
"type": "STRICT_DNS"
}

admin:
access_log_path: "/dev/null"
address:
socket_address:
address: 127.0.0.1
port_value: 49000
```

nomad job config:
```
job "envoy" {
datacenters = ["dc1", "server-dc"]

constraint {
distinct_hosts = true
}

group "envoy" {
count = "1"
network {
port "http" {}
port "https" {}
port "lb" {}
}

task "envoy" {
driver = "docker"
config {
network_mode = "host"
mount {
type = "bind"
source = "/etc/envoy/envoy.yml"
target = "/etc/envoy/envoy.yaml"
readonly = false
}
mount {
type = "bind"
source = "/etc/envoy/xds"
target = "/etc/envoy/xds"
readonly = false
}
mount {
type = "bind"
source = "/certs"
target = "/etc/envoy/certs"
readonly = false
}
image = "envoyproxy/envoy-contrib:v1.27-latest"
command = "--config-path /etc/envoy/envoy.yaml"
}

env {
COMPOSE_HTTP_TIMEOUT = "120"
ENVOY_UID=0
}

restart {
attempts = 3
interval = "10m"
delay = "10s"
mode = "delay"
}

resources {
cpu = 512
memory = 1024
}

service {
provider = "consul"
name = "envoy"
check {
name = "envoy-443"
type = "tcp"
port = "443"
address_mode = "driver"
interval = "15s"
timeout = "5s"
}
}
}
}
}
```

Start with the Consul xDS gRPC external agent path and the Envoy Aggregated Discovery Service configuration shown in the report. Reproduce the setup with Consul 1.17.0, Envoy 1.27, and Nomad 1.6.1, then trace why StreamAggregatedResources closes with “not implemented.” Done means Envoy receives the xDS resources and creates the expected listener without the repeated warnings.

Written by the indexing model from the issue text.