switching from docker to kaniko in different profiles is hard to configure

Today, if you want to switch _all_ your builder types to kaniko from local docker, you'll have to repeat all your artifacts. This is cumbersome so people resort to using YamlAnchors. E.g. https://github.com/ForgeRock/forgeops/blob/master/skaffold.yaml

The hydrated version of this yaml is this:

```
apiVersion: skaffold/v2beta5
kind: Config
build:
artifacts:
- image: am
context: docker/7.0/am
docker:
dockerfile: Dockerfile
- image: amster
context: docker/7.0/amster
docker:
dockerfile: Dockerfile
- image: idm
context: docker/7.0/idm
docker:
dockerfile: Dockerfile
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
docker:
dockerfile: Dockerfile
- image: ig
context: docker/7.0/ig
docker:
dockerfile: Dockerfile
tagPolicy:
sha256: {}
local:
concurrency: 1
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/all
statusCheckDeadlineSeconds: 600
profiles:
- name: forgeops
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/forgeops
- name: nightly
build:
artifacts:
- image: am
context: docker/7.0/am
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: amster
context: docker/7.0/amster
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: idm
context: docker/7.0/idm
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-cts
context: docker/7.0/ds
kaniko:
flags:
- --single-snapshot
dockerfile: cts/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-idrepo
context: docker/7.0/ds/
kaniko:
flags:
- --single-snapshot
dockerfile: idrepo/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: forgeops-secrets
context: docker/forgeops-secrets
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
tagPolicy:
sha256: {}
cluster:
pullSecretName: kaniko-secret
namespace: kaniko
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/nightly
- name: smoke
build:
artifacts:
- image: am
context: docker/7.0/am
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: amster
context: docker/7.0/amster
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: idm
context: docker/7.0/idm
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-cts
context: docker/7.0/ds
kaniko:
flags:
- --single-snapshot
dockerfile: cts/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-idrepo
context: docker/7.0/ds/
kaniko:
flags:
- --single-snapshot
dockerfile: idrepo/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: forgeops-secrets
context: docker/forgeops-secrets
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
tagPolicy:
gitCommit: {}
cluster:
pullSecretName: kaniko-secret
namespace: kaniko
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/smoke
- name: default-kaniko
build:
artifacts:
- image: am
context: docker/7.0/am
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: amster
context: docker/7.0/amster
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: idm
context: docker/7.0/idm
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-cts
context: docker/7.0/ds
kaniko:
flags:
- --single-snapshot
dockerfile: cts/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-idrepo
context: docker/7.0/ds/
kaniko:
flags:
- --single-snapshot
dockerfile: idrepo/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: forgeops-secrets
context: docker/forgeops-secrets
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
tagPolicy:
sha256: {}
cluster:
pullSecretName: kaniko-secret
namespace: kaniko
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/all
statusCheckDeadlineSeconds: 600
- name: kdev
build:
artifacts:
- image: am
context: docker/7.0/am
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: amster
context: docker/7.0/amster
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: idm
context: docker/7.0/idm
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-cts
context: docker/7.0/ds
kaniko:
flags:
- --single-snapshot
dockerfile: cts/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: ds-idrepo
context: docker/7.0/ds/
kaniko:
flags:
- --single-snapshot
dockerfile: idrepo/Dockerfile
image: gcr.io/kaniko-project/executor:v0.20.0
- image: forgeops-secrets
context: docker/forgeops-secrets
kaniko:
flags:
- --single-snapshot
image: gcr.io/kaniko-project/executor:v0.20.0
tagPolicy:
sha256: {}
cluster:
pullSecretName: kaniko-secret
namespace: kaniko
deploy:
kustomize:
paths:
- ./dev
- name: security
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/security
- name: small
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/small
- name: medium
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/medium
- name: large
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/large
- name: platform-deployment
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: idm
context: docker/7.0/idm
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/lodestar
- name: ig-only
build:
artifacts:
- image: ig
context: docker/7.0/ig
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/ig-only
- name: am-only
build:
artifacts:
- image: am
context: docker/7.0/am
- image: amster
context: docker/7.0/amster
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
tagPolicy:
gitCommit: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/am-only
- name: idm-only
build:
artifacts:
- image: idm
context: docker/7.0/idm
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/idm-only
- name: ds-only
build:
artifacts:
- image: ds-cts
context: docker/7.0/ds
docker:
dockerfile: cts/Dockerfile
- image: ds-idrepo
context: docker/7.0/ds/
docker:
dockerfile: idrepo/Dockerfile
- image: forgeops-secrets
context: docker/forgeops-secrets
tagPolicy:
sha256: {}
deploy:
kustomize:
paths:
- ./kustomize/overlay/7.0/ds-only

```

A smaller example:

```
apiVersion: skaffold/v2beta5
kind: Config
build:
artifacts:
- image: skaffold-example
local: {}
deploy:
kubectl:
manifests:
- k8s-*
profiles:
- name: build-with-kaniko
build:
artifacts:
- image: skaffold-example
kaniko:
image: gcr.io/kaniko-project/executor:v0.20.0
flags:
- '--single-snapshot'
cluster: {}
```

Issues with our current config reflecting in repetition are:
- `kaniko.image`, `kaniko.flags`, etc. are only available on the artifact level. **Recommendation:** If these would be available on the `build.cluster.kaniko` level, one wouldn't have to repeat them for each artifact
- if I specify `artifacts.kaniko.image` and try to build with `local` docker, then skaffold fails with:

```
invalid skaffold config: found a 'kaniko' artifact, which is incompatible with the 'local' builder:

image: skaffold-example
context: .
kaniko:
flags:
- --single-snapshot
dockerfile: Dockerfile
initImage: busybox
image: gcr.io/kaniko-project/executor:v0.20.0

To use the 'kaniko' builder, add the 'cluster' stanza to the 'build' section of your configuration. For information, see https://skaffold.dev/docs/pipeline-stages/builders/
```
**Recommendation**: skaffold could ignore kaniko related settings on the artifacts when building with docker. This means that artifacts could have _at the same time_ settings for docker, kaniko and GCB (though this last one doesn't exist). In goland this might translate to an artifact being able to become simultaneously a KanikoArtifact and a DockerArtifact - not sure - but our abstraction here is messy.

The final result would be something like this:

```
apiVersion: skaffold/v2beta5
kind: Config
build:
artifacts:
- image: skaffold-example
docker: {} # mentioned for explicitness
kaniko: {} # mentioned for explicitness
local: {}
deploy:
kubectl:
manifests:
- k8s-*
profiles:
- name: build-with-kaniko
build:
cluster:
kaniko:
image: gcr.io/kaniko-project/executor:v0.20.0
flags:
- '--single-snapshot'
```