`consul connect envoy` using `default` token instead of specified token
- Dominant language
- Go
- Stars
- 30.1k
- Forks
- 4.6k
- Avg merge
- 1d 18h
- Merged PRs (30d)
- 39
Description
#### Overview of the Issue
Removing `service:write` from `default` token is causing errors on `consul connect envoy`, although a separate token with `service:write` is specified for the command.
---
#### Reproduction Steps
1. Start with a `default` token with write permissions for everything.
2. Start `consul connect envoy -sidecar-for=service` or `consul connect envoy -gateway=ingress` with an ACL token with the following policy:
```
agent_prefix "" {
policy = "read"
}
service_prefix "" {
policy = "write"
}
mesh = "write"
```
3. Apply this restrictive policy to the `default` token:
```
node_prefix "" {
policy = "read"
}
operator = "read"
query_prefix "" {
policy = "read"
}
service_prefix "" {
policy = "read"
}
```
4. Observe errors indicating the `default` token is being used instead of the token specified for the envoy proxy.
### Consul info for both Client and Server
Client info
We run a custom fork of consul that has additional logging and adjusts some timeouts, no behavioral changes related to ACLs:
```
agent:
check_monitors = 2
check_ttls = 0
checks = 8
services = 7
build:
prerelease = company
revision =
version = 1.20.1
version_metadata =
consul:
acl = enabled
known_servers = 3
server = false
runtime:
arch = amd64
cpu_count = 2
goroutines = 211
max_procs = 2
os = linux
version = go1.22.5
serf_lan:
coordinate_resets = 0
encrypted = false
event_queue = 0
event_time = 472
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 588665
members = 166
query_queue = 0
query_time = 1
```
```
{
"acl": {
"default_policy": "deny",
"down_policy": "async-cache",
"enable_token_persistence": true,
"enable_token_replication": true,
"enabled": true,
"tokens": {
"agent": "token",
"config_file_service_registration": "token",
"default": "token"
}
},
"addresses": {
"dns": "127.0.0.1",
"grpc": "127.0.0.1",
"http": "127.0.0.1"
},
"advertise_addr": "{{ GetInterfaceIP \"ens4\" }}",
"auto_reload_config": true,
"config_entries": {
},
"connect": {
"ca_config": {
"csr_max_concurrent": 0,
"csr_max_per_second": 50,
"leaf_cert_ttl": "72h"
},
"enabled": true
},
"data_dir": "/opt/consul",
"datacenter": "datacenter-2",
"default_intention_policy": "allow",
"disable_remote_exec": true,
"disable_update_check": true,
"dns_config": {
"allow_stale": true,
"max_stale": "87600h",
"service_ttl": {
"*": "2s",
"service-name": "5s",
"service-name": "5s",
"service-name": "5s",
"service-name": "10s",
"service-name": "2s",
"service-name": "5s"
}
},
"enable_central_service_config": true,
"enable_script_checks": true,
"enable_syslog": true,
"log_level": "INFO",
"node_meta": {
"cluster": "cluster",
"machine_type": "machine-type",
"zone": "zone"
},
"node_name": "node-name",
"ports": {
"dns": 8600,
"grpc": 8502,
"http": 8500,
"serf_lan": 8301,
"serf_wan": 8302
},
"primary_datacenter": "datacenter",
"retry_join": [
"provider=gce tag_value=value zone_pattern=us-central1-.*"
],
"retry_join_wan": [
],
"tls": {
"defaults": {
"ca_file": "/etc/consul/ssl/ca.pem",
"tls_min_version": "TLSv1_2",
"verify_incoming": false,
"verify_outgoing": false
}
},
"use_streaming_backend": false
}
```
Server info
```
agent:
check_monitors = 2
check_ttls = 0
checks = 6
services = 5
build:
prerelease = company
revision =
version = 1.20.1
version_metadata =
consul:
acl = enabled
bootstrap = false
known_datacenters = 5
leader = true
leader_addr = leader_ip:8300
server = true
raft:
applied_index = 1363425798
commit_index = 1363425798
fsm_pending = 0
last_contact = 0
last_log_index = 1363425798
last_log_term = 872
last_snapshot_index = 1363414104
last_snapshot_term = 872
latest_configuration = [{Suffrage:Voter ID:id1 Address:leader_ip:8300} {Suffrage:Voter ID:id2 Address:second_ip:8300} {Suffrage:Voter ID:id3 Address:third_ip:8300}]
latest_configuration_index = 0
num_peers = 2
protocol_version = 3
protocol_version_max = 3
protocol_version_min = 0
snapshot_version_max = 1
snapshot_version_min = 0
state = Leader
term = 872
runtime:
arch = amd64
cpu_count = 2
goroutines = 5059
max_procs = 2
os = linux
version = go1.22.5
serf_lan:
coordinate_resets = 0
encrypted = false
event_queue = 0
event_time = 472
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 588665
members = 166
query_queue = 0
query_time = 1
serf_wan:
coordinate_resets = 0
encrypted = false
event_queue = 0
event_time = 1
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 34563
members = 15
query_queue = 0
query_time = 1
```
```
{
"acl": {
"default_policy": "deny",
"down_policy": "async-cache",
"enable_token_persistence": true,
"enable_token_replication": true,
"enabled": true,
"tokens": {
"agent": "token",
"config_file_service_registration": "token",
"default": "token",
"initial_management": "token",
"replication": "token"
}
},
"addresses": {
"dns": "0.0.0.0",
"grpc": "0.0.0.0",
"grpc_tls": "0.0.0.0",
"http": "0.0.0.0",
"https": "0.0.0.0"
},
"advertise_addr": "{{ GetInterfaceIP \"ens4\" }}",
"auto_reload_config": true,
"bootstrap_expect": 3,
"config_entries": {
"bootstrap": [
{
"config": {
"protocol": "http"
},
"kind": "proxy-defaults",
"name": "global"
},
{
"kind": "service-defaults",
"name": "http-1",
"protocol": "http"
},
{
"kind": "service-defaults",
"name": "grpc",
"protocol": "grpc"
},
{
"kind": "service-router",
"name": "http-1",
"routes": [
]
},
{
"kind": "service-router",
"name": "grpc",
"routes": [
]
}
]
},
"connect": {
"ca_config": {
"csr_max_concurrent": 2,
"csr_max_per_second": 0,
"leaf_cert_ttl": "72h"
},
"enabled": true
},
"data_dir": "/opt/consul",
"datacenter": "datacenter",
"default_intention_policy": "allow",
"disable_remote_exec": true,
"disable_update_check": true,
"dns_config": {
"allow_stale": true,
"max_stale": "87600h",
"service_ttl": {
"*": "2s",
"service_name": "5s",
"service_name": "5s",
"service_name": "5s",
"service_name": "10s",
"service_name": "2s",
"service_name": "5s"
}
},
"enable_central_service_config": true,
"enable_script_checks": true,
"enable_syslog": true,
"leave_on_terminate": true,
"log_acl_info": true,
"log_level": "INFO",
"node_meta": {
"cluster": "cluster",
"machine_type": "machine_type",
"zone": "zone"
},
"node_name": "node_name",
"ports": {
"dns": 8600,
"grpc": 8502,
"grpc_tls": 8503,
"http": 8500,
"https": 8501,
"serf_lan": 8301,
"serf_wan": 8302
},
"primary_datacenter": "datacenter",
"retry_join": [
"provider=gce tag_value=tag_value zone_pattern=us-central1-.*"
],
"retry_join_wan": [
],
"server": true,
"skip_leave_on_interrupt": false,
"telemetry": {
"disable_hostname": true,
"prometheus_retention_time": "40s"
},
"tls": {
"defaults": {
"ca_file": "/etc/consul/ssl/ca.pem",
"cert_file": "/etc/consul/ssl/cert.pem",
"key_file": "/etc/consul/ssl/key.pem",
"tls_min_version": "TLSv1_2",
"verify_incoming": false,
"verify_outgoing": false
}
},
"ui": true,
"use_streaming_backend": false
}
```
### Operating system and Environment details
x86 debian 12, gcp
### Log Fragments
Instance running `consul connect envoy -sidecar-for=service-one`:
```
agent.cache: handling error in Cache.Notify: cache-type=trust-bundles error="rpc error: code = Unknown desc = Permission denied: token with AccessorID '' lacks permission 'service:write' on \"service-one\"" index=674388994
```
Instance running `consul connect envoy -gateway=ingress`
```
agent.client: RPC failed to server: method=ConnectCA.Sign server=:8300 error="rpc error making call: rpc error making call: Permission denied: token with AccessorID '' lacks permission 'service:write' on \"redacted-ingress-name\""
```
Contributor guide
Research direction
Start by tracing the `consul connect envoy` command's token handling for the `-sidecar-for` and `-gateway` paths, then reproduce the ACL failures described in the issue. Done means each command uses the explicitly supplied token rather than the default token, with the provided restrictive default-token policy no longer causing `service:write` errors.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- go
- Domain
- authentication, cli
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 35/100