Nomad only pull images supported by defined helper
- Dominant language
- Go
- Stars
- 17k
- Forks
- 2.1k
- Avg merge
- 1d 9h
- Merged PRs (30d)
- 105
Description
### Nomad version
Output from `nomad version`
Nomad v0.12.7 (6147cb578794cb2d0c35d68fe1791728a09bb081)
### Operating system and Environment details
> cat /etc/os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"
CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"
### Issue
I have my Nomad running on an on-prem machine and using ECR as a container registry. For a long time, I was setting up credentials helper so it would pull the images from the ECR. Once that was done, everything worked for a while. Until I realized, that once the image of envoyproxy/envoy was gone, sidecar for any service wouldn't start with an error: `Failed to find docker auth for repo "envoyproxy/envoy": docker-credential-ecr-login with input "envoyproxy/envoy" failed with stderr: exit status 1`. Clearly, for some reason, it tries to authenticate against docker.io with ecr-login cred helper.
### Reproduction steps
nomad.hcl
```
data_dir = "/opt/nomad/data"
bind_addr = "0.0.0.0"
server {
enabled = true
bootstrap_expect = 1
}
client {
enabled = true
servers = ["127.0.0.1:4646"]
host_volume "postgres" {
path = "/opt/postgres/data"
read_only = false
}
options = {
"docker.auth.config" = "/etc/docker/config.json"
"docker.auth.helper" = "ecr-login"
}
}
consul {
address = "127.0.0.1:8500"
server_service_name = "nomad"
client_service_name = "nomad-client"
auto_advertise = true
server_auto_join = true
client_auto_join = true
}
```
config.json
```
{
"credHelpers": {
"059899322608.dkr.ecr.us-east-1.amazonaws.com": "ecr-login"
},
"credsStore": "ecr-login"
}
```
### Job file (if appropriate)
```
job "my_job" {
datacenters = ["dc1"]
type = "service"
update {
max_parallel = 1
min_healthy_time = "10s"
healthy_deadline = "3m"
progress_deadline = "10m"
auto_revert = false
canary = 0
}
migrate {
max_parallel = 1
health_check = "checks"
min_healthy_time = "10s"
healthy_deadline = "5m"
}
group "postgres" {
network {
mode = "bridge"
}
volume "postgres" {
type = "host"
read_only = false
source = "postgres"
}
restart {
attempts = 3
interval = "15m"
delay = "15s"
mode = "delay"
}
reschedule {
attempts = 15
interval = "1h"
delay = "30s"
delay_function = "exponential"
max_delay = "120s"
unlimited = false
}
service {
name = "postgres"
tags = ["infra", "db"]
port = 5432
connect {
sidecar_service {}
}
check {
type = "script"
command = "pg_isready"
interval = "60s"
timeout = "5s"
task = "postgres"
check_restart {
limit = 3
grace = "60s"
ignore_warnings = false
}
}
}
task "postgres" {
driver = "docker"
volume_mount {
volume = "postgres"
destination = "/var/lib/postgresql/data"
read_only = false
}
config {
image = "postgres:12"
}
env {
POSTGRES_DB = ""
POSTGRES_USER = ""
POSTGRES_PASSWORD = ""
}
resources {
cpu = 500
memory = 1024
network {
mbits = 10
}
}
}
}
group "my_task_group" {
network {
mode = "bridge"
port "my_service_healthcheck" {
to = -1
}
}
restart {
attempts = 18
interval = "5m"
delay = "15s"
mode = "delay"
}
reschedule {
attempts = 15
interval = "1h"
delay = "30s"
delay_function = "exponential"
max_delay = "120s"
unlimited = false
}
service {
name = "my_service"
tags = ["microservice"]
port = 8030
check {
name = "my-service-health"
type = "http"
port = "my_service_healthcheck"
path = "/v1/health"
interval = "10s"
timeout = "5s"
check_restart {
limit = 3
grace = "60s"
ignore_warnings = false
}
}
connect {
sidecar_service {
proxy {
expose {
path {
path = "/v1/health"
protocol = "http"
local_path_port = 8030
listener_port = "my_service_healthcheck"
}
}
upstreams = {
destination_name = "postgres"
local_bind_port = 5432
}
}
}
}
}
task "my_task" {
driver = "docker"
config {
image = "____ACCOUNT_ID____.dkr.ecr.____REGION____.amazonaws.com/image-name:tag"
force_pull = true
}
env {
DS_DB_NAME = ""
DS_DB_USER = ""
DS_DB_PASSWORD = ""
}
resources {
cpu = 500
memory = 512
network {
mbits = 10
}
}
}
}
}
```
### Nomad Client/Server logs (if appropriate)
Nothing that would be related to this issue...
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.