Empty rendered file after Consul service restart
- Dominant language
- Go
- Stars
- 30.1k
- Forks
- 4.6k
- Avg merge
- 2d 6h
- Merged PRs (30d)
- 43
Description
#### Overview of the Issue
I am using Consul Template to render configuration file based of the content in Consul Service meta tags.
Everything works correct but I have noticed that after consul service restart, configuration file is overwritten with empty data.
I believe that this happens because given node has not registered to Consul cluster yet, and returns empty value for consul-template client.
Configuration file is updated with correct data when the configured minimum wait time for consul-template kicks in.
However, even if the minimum wait time is configured for one second it can still result as service outage, depending of the content of configuration file.
---
#### Reproduction Steps
Steps to reproduce this issue, eg:
1. Create a cluster with 1 client nodes and 3 server nodes
2. Configure consul-template to to pull data from Consul Service meta keys as following:
```
Consul-template:
### Consul Template Configuration
```hcl
consul {
address = "127.0.0.1:8500"
token = "xxxxxxxxx"
retry {
enabled = true
attempts = 12
backoff = "250ms"
max_backoff = "1m"
}
ssl {
enabled = false
verify = true
}
}
reload_signal = "SIGHUP"
kill_signal = "SIGINT"
log_level = "err"
pid_file = "/var/run/consul-template/consul-template.pid"
wait {
min = "5s"
max = "10s"
}
template {
source = "/etc/consul-template/templates/serviceidentity_app01-2019052800.tpl"
destination = "/app_deployment/app01/central/dbaccess/serveridentity.txt"
create_dest_dirs = true
command = "bash -c 'if id -u majandust > /dev/null 2>&1; then chown majandust:root /app_deployment/app01/central/dbaccess/serveridentity.txt; fi'"
command_timeout = "60s"
error_on_missing_key = false
perms = 0644
backup = true
wait {
min = "2s"
max = "10s"
}
}
```
/etc/consul-template/templates/serviceidentity_app01-2019052800.tpl file content:
`{{ with node }}{{ range services }}{{ $name := .Name }}{{ $service := service (printf "%s|any" $name) }}{{ range $service }}{{ if eq node.Node.Node .Node }}{{ $ews_instance := index .ServiceMeta "ews_instance" }}{{ if eq $ews_instance "app01" }}{{ index .ServiceMeta "serveridentity" }}{{ end }}{{ end }}{{ end }}{{ end }}{{ end -}}`
3. Restart consul service in client node, configuration file is overwritten with empty data.
### Consul info for both Client and Server
Client info
Consul version: 1.12.9
```
agent:
check_monitors = 0
check_ttls = 0
checks = 0
services = 3
build:
prerelease =
revision = 96505027
version = 1.12.9
version_metadata =
consul:
acl = enabled
known_servers = 3
server = false
runtime:
arch = amd64
cpu_count = 16
goroutines = 355
max_procs = 16
os = linux
version = go1.18.9
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 172
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2349
members = 47
query_queue = 0
query_time = 1
```
```
agent:
check_monitors = 0
check_ttls = 0
checks = 0
services = 0
build:
prerelease =
revision = 96505027
version = 1.12.9
version_metadata =
consul:
acl = enabled
bootstrap = false
known_datacenters = 6
leader = true
leader_addr = 10.0.30.100:8300
server = true
raft:
applied_index = 20156753
commit_index = 20156753
fsm_pending = 0
last_contact = 0
last_log_index = 20156753
last_log_term = 342
last_snapshot_index = 20153060
last_snapshot_term = 341
latest_configuration = [{Suffrage:Voter ID:e6b06d93-5a8e-03cf-de20-3f182da2b211 Address:10.0.30.100:8300} {Suffrage:Voter ID:95f073c3-2be5-bc28-f7c7-52d77af1c2c0 Address:10.0.130.100:8300} {Suffrage:Voter ID:db4dcaa1-7506-965e-0bf4-e008f932f340 Address:10.0.230.100:8300}]
latest_configuration_index = 0
num_peers = 2
protocol_version = 3
protocol_version_max = 3
protocol_version_min = 0
snapshot_version_max = 1
snapshot_version_min = 0
state = Leader
term = 342
runtime:
arch = amd64
cpu_count = 2
goroutines = 1129
max_procs = 2
os = linux
version = go1.18.9
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 172
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2349
members = 47
query_queue = 0
query_time = 1
serf_wan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 1
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2129
members = 18
query_queue = 0
query_time = 1
```
Server info
Consul version: 1.12.9
```
{
"acl": {
"default_policy": "deny",
"down_policy": "extend-cache",
"enable_token_replication": true,
"enabled": true,
"token_ttl": "30s",
"tokens": {
"agent": "xxx",
"initial_management": "null",
"replication": "xxx"
}
},
"addresses": {
"dns": "0.0.0.0",
"grpc": "127.0.0.1",
"http": "127.0.0.1",
"https": "0.0.0.0"
},
"advertise_addr": "10.0.30.100",
"advertise_addr_wan": "x.x.x.x",
"bind_addr": "10.0.30.100",
"bootstrap": false,
"bootstrap_expect": 3,
"ca_file": "/etc/consul/ssl/domain.crt",
"cert_file": "/etc/consul/ssl/domain.com.crt",
"client_addr": "127.0.0.1",
"data_dir": "/var/consul",
"datacenter": "us-east-2",
"disable_keyring_file": true,
"disable_update_check": false,
"enable_local_script_checks": false,
"enable_script_checks": false,
"encrypt": "xxx",
"key_file": "/etc/consul/ssl/domain.com.key",
"limits": {
"http_max_conns_per_client": 1000
},
"log_file": "/var/log/consul/consul.log",
"log_level": "INFO",
"log_rotate_bytes": 1000000,
"log_rotate_duration": "24h",
"log_rotate_max_files": 7,
"node_meta": {},
"node_name": "consul-srv-us-east-2a-1",
"performance": {
"leave_drain_time": "5s",
"raft_multiplier": 1,
"rpc_hold_timeout": "7s"
},
"ports": {
"dns": 8600,
"grpc": -1,
"http": -1,
"https": 8443,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
},
"primary_datacenter": "eu-central-1",
"raft_protocol": 3,
"retry_interval": "30s",
"retry_interval_wan": "30s",
"retry_join": [
"10.0.30.100",
"10.0.130.100",
"10.0.230.100"
],
"retry_join_wan": [
"x.x.x.x",
"x.x.x.x",
"x.x.x.x",
],
"retry_max": 0,
"retry_max_wan": 0,
"server": true,
"ui": true,
"verify_incoming": false,
"verify_incoming_https": false,
"verify_outgoing": true,
"verify_server_hostname": false
```
```
{
"acl": {
"default_policy": "deny",
"down_policy": "extend-cache",
"enabled": true,
"token_ttl": "30s",
"tokens": {
"agent": "xxx",
"default": "xxx"
}
},
"addresses": {
"dns": "127.0.0.1",
"grpc": "127.0.0.1",
"http": "127.0.0.1",
"https": "127.0.0.1"
},
"advertise_addr": "10.0.20.10",
"advertise_addr_wan": "10.0.20.10",
"bind_addr": "10.0.20.10",
"ca_file": "/etc/consul/ssl/domain.crt",
"cert_file": "/etc/consul/ssl/domain.crt",
"client_addr": "127.0.0.1",
"data_dir": "/var/consul",
"datacenter": "us-east-2",
"disable_keyring_file": true,
"disable_update_check": false,
"enable_local_script_checks": false,
"enable_script_checks": false,
"encrypt": "xxxx,
"key_file": "/etc/consul/ssl/domain.com.key",
"limits": {
"http_max_conns_per_client": 1000
},
"log_file": "/var/log/consul/consul.log",
"log_level": "INFO",
"log_rotate_bytes": 1000000,
"log_rotate_duration": "24h",
"log_rotate_max_files": 7,
"node_meta": {
"cluster": "us.domain.com"
},
"node_name": "app-us-east-2a-1-prod",
"performance": {
"leave_drain_time": "5s",
"raft_multiplier": 1,
"rpc_hold_timeout": "7s"
},
"ports": {
"dns": 8600,
"grpc": -1,
"http": 8500,
"https": 8443,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
},
"primary_datacenter": "eu-central-1",
"raft_protocol": 3,
"retry_interval": "30s",
"retry_join": [
"10.0.30.100",
"10.0.130.100",
"10.0.230.100"
],
"retry_max": 0,
"server": false,
"ui": false,
"verify_incoming": false,
"verify_incoming_https": false,
"verify_outgoing": true,
"verify_server_hostname": false
```
### Operating system and Environment details
All consul services are running on Amazon ec2 nodes on Centos 7.
Contributor guide
Assessment
This issue has not been assessed yet.