hashicorp / hashicorp/consul

Empty rendered file after Consul service restart

Open
#16,539 0 comments 2 reactions 0 assignees View on GitHub
Dominant language
Go
Stars
30.1k
Forks
4.6k
Avg merge
2d 6h
Merged PRs (30d)
43

Description

#### Overview of the Issue

I am using Consul Template to render configuration file based of the content in Consul Service meta tags.
Everything works correct but I have noticed that after consul service restart, configuration file is overwritten with empty data.

I believe that this happens because given node has not registered to Consul cluster yet, and returns empty value for consul-template client.

Configuration file is updated with correct data when the configured minimum wait time for consul-template kicks in.
However, even if the minimum wait time is configured for one second it can still result as service outage, depending of the content of configuration file.

---

#### Reproduction Steps

Steps to reproduce this issue, eg:

1. Create a cluster with 1 client nodes and 3 server nodes
2. Configure consul-template to to pull data from Consul Service meta keys as following:
```
Consul-template:
### Consul Template Configuration

```hcl
consul {
address = "127.0.0.1:8500"
token = "xxxxxxxxx"
retry {

enabled = true
attempts = 12
backoff = "250ms"
max_backoff = "1m"
}

ssl {

enabled = false
verify = true
}
}

reload_signal = "SIGHUP"
kill_signal = "SIGINT"
log_level = "err"
pid_file = "/var/run/consul-template/consul-template.pid"
wait {
min = "5s"
max = "10s"
}

template {
source = "/etc/consul-template/templates/serviceidentity_app01-2019052800.tpl"
destination = "/app_deployment/app01/central/dbaccess/serveridentity.txt"
create_dest_dirs = true
command = "bash -c 'if id -u majandust > /dev/null 2>&1; then chown majandust:root /app_deployment/app01/central/dbaccess/serveridentity.txt; fi'"
command_timeout = "60s"
error_on_missing_key = false
perms = 0644
backup = true
wait {
min = "2s"
max = "10s"
}
}

```
/etc/consul-template/templates/serviceidentity_app01-2019052800.tpl file content:

`{{ with node }}{{ range services }}{{ $name := .Name }}{{ $service := service (printf "%s|any" $name) }}{{ range $service }}{{ if eq node.Node.Node .Node }}{{ $ews_instance := index .ServiceMeta "ews_instance" }}{{ if eq $ews_instance "app01" }}{{ index .ServiceMeta "serveridentity" }}{{ end }}{{ end }}{{ end }}{{ end }}{{ end -}}`

3. Restart consul service in client node, configuration file is overwritten with empty data.

### Consul info for both Client and Server

Client info

Consul version: 1.12.9

```
agent:
check_monitors = 0
check_ttls = 0
checks = 0
services = 3
build:
prerelease =
revision = 96505027
version = 1.12.9
version_metadata =
consul:
acl = enabled
known_servers = 3
server = false
runtime:
arch = amd64
cpu_count = 16
goroutines = 355
max_procs = 16
os = linux
version = go1.18.9
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 172
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2349
members = 47
query_queue = 0
query_time = 1
```

```
agent:
check_monitors = 0
check_ttls = 0
checks = 0
services = 0
build:
prerelease =
revision = 96505027
version = 1.12.9
version_metadata =
consul:
acl = enabled
bootstrap = false
known_datacenters = 6
leader = true
leader_addr = 10.0.30.100:8300
server = true
raft:
applied_index = 20156753
commit_index = 20156753
fsm_pending = 0
last_contact = 0
last_log_index = 20156753
last_log_term = 342
last_snapshot_index = 20153060
last_snapshot_term = 341
latest_configuration = [{Suffrage:Voter ID:e6b06d93-5a8e-03cf-de20-3f182da2b211 Address:10.0.30.100:8300} {Suffrage:Voter ID:95f073c3-2be5-bc28-f7c7-52d77af1c2c0 Address:10.0.130.100:8300} {Suffrage:Voter ID:db4dcaa1-7506-965e-0bf4-e008f932f340 Address:10.0.230.100:8300}]
latest_configuration_index = 0
num_peers = 2
protocol_version = 3
protocol_version_max = 3
protocol_version_min = 0
snapshot_version_max = 1
snapshot_version_min = 0
state = Leader
term = 342
runtime:
arch = amd64
cpu_count = 2
goroutines = 1129
max_procs = 2
os = linux
version = go1.18.9
serf_lan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 172
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2349
members = 47
query_queue = 0
query_time = 1
serf_wan:
coordinate_resets = 0
encrypted = true
event_queue = 0
event_time = 1
failed = 0
health_score = 0
intent_queue = 0
left = 0
member_time = 2129
members = 18
query_queue = 0
query_time = 1
```

Server info

Consul version: 1.12.9

```
{
"acl": {
"default_policy": "deny",
"down_policy": "extend-cache",
"enable_token_replication": true,
"enabled": true,
"token_ttl": "30s",
"tokens": {
"agent": "xxx",
"initial_management": "null",
"replication": "xxx"
}
},
"addresses": {
"dns": "0.0.0.0",
"grpc": "127.0.0.1",
"http": "127.0.0.1",
"https": "0.0.0.0"
},
"advertise_addr": "10.0.30.100",
"advertise_addr_wan": "x.x.x.x",
"bind_addr": "10.0.30.100",
"bootstrap": false,
"bootstrap_expect": 3,
"ca_file": "/etc/consul/ssl/domain.crt",
"cert_file": "/etc/consul/ssl/domain.com.crt",
"client_addr": "127.0.0.1",
"data_dir": "/var/consul",
"datacenter": "us-east-2",
"disable_keyring_file": true,
"disable_update_check": false,
"enable_local_script_checks": false,
"enable_script_checks": false,
"encrypt": "xxx",
"key_file": "/etc/consul/ssl/domain.com.key",
"limits": {
"http_max_conns_per_client": 1000
},
"log_file": "/var/log/consul/consul.log",
"log_level": "INFO",
"log_rotate_bytes": 1000000,
"log_rotate_duration": "24h",
"log_rotate_max_files": 7,
"node_meta": {},
"node_name": "consul-srv-us-east-2a-1",
"performance": {
"leave_drain_time": "5s",
"raft_multiplier": 1,
"rpc_hold_timeout": "7s"
},
"ports": {
"dns": 8600,
"grpc": -1,
"http": -1,
"https": 8443,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
},
"primary_datacenter": "eu-central-1",
"raft_protocol": 3,
"retry_interval": "30s",
"retry_interval_wan": "30s",
"retry_join": [
"10.0.30.100",
"10.0.130.100",
"10.0.230.100"
],
"retry_join_wan": [
"x.x.x.x",
"x.x.x.x",
"x.x.x.x",
],
"retry_max": 0,
"retry_max_wan": 0,
"server": true,
"ui": true,
"verify_incoming": false,
"verify_incoming_https": false,
"verify_outgoing": true,
"verify_server_hostname": false
```

```
{
"acl": {
"default_policy": "deny",
"down_policy": "extend-cache",
"enabled": true,
"token_ttl": "30s",
"tokens": {
"agent": "xxx",
"default": "xxx"
}
},
"addresses": {
"dns": "127.0.0.1",
"grpc": "127.0.0.1",
"http": "127.0.0.1",
"https": "127.0.0.1"
},
"advertise_addr": "10.0.20.10",
"advertise_addr_wan": "10.0.20.10",
"bind_addr": "10.0.20.10",
"ca_file": "/etc/consul/ssl/domain.crt",
"cert_file": "/etc/consul/ssl/domain.crt",
"client_addr": "127.0.0.1",
"data_dir": "/var/consul",
"datacenter": "us-east-2",
"disable_keyring_file": true,
"disable_update_check": false,
"enable_local_script_checks": false,
"enable_script_checks": false,
"encrypt": "xxxx,
"key_file": "/etc/consul/ssl/domain.com.key",
"limits": {
"http_max_conns_per_client": 1000
},
"log_file": "/var/log/consul/consul.log",
"log_level": "INFO",
"log_rotate_bytes": 1000000,
"log_rotate_duration": "24h",
"log_rotate_max_files": 7,
"node_meta": {
"cluster": "us.domain.com"
},
"node_name": "app-us-east-2a-1-prod",
"performance": {
"leave_drain_time": "5s",
"raft_multiplier": 1,
"rpc_hold_timeout": "7s"
},
"ports": {
"dns": 8600,
"grpc": -1,
"http": 8500,
"https": 8443,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
},
"primary_datacenter": "eu-central-1",
"raft_protocol": 3,
"retry_interval": "30s",
"retry_join": [
"10.0.30.100",
"10.0.130.100",
"10.0.230.100"
],
"retry_max": 0,
"server": false,
"ui": false,
"verify_incoming": false,
"verify_incoming_https": false,
"verify_outgoing": true,
"verify_server_hostname": false
```

### Operating system and Environment details
All consul services are running on Amazon ec2 nodes on Centos 7.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.