aws-samples / aws-samples/cdk-eks-karpenter
Stack removal fails
- Dominant language
- TypeScript
- Stars
- 48
- Forks
- 21
- PR merge metrics
- No merged PRs in 30d
Description
Hey,
we have been using `cdk-eks-karpenter` for a while now and we have been experiencing issues with the removal of stacks where karpenter has been installed using this package. Basically CloudFormation triggers the delete on the CustomResource which installed the yaml file into the cluster that then fails / times out. In the EKS console all the nodes have already been removed and the cluster is pretty much only still existing on paper (but I cannot connect with kubectl to it anymore). Eventually the CustomResource times out after 1h and CloudFormation fails.
We have produced this sort of minimal example where the error still occurs and where we do nothing more than just creating a cluster within our pre-created VPC and then install karpenter using this package.
```typescript
import { CONFIG } from '@/src/config';
import { vpcName } from '@/src/utils';
import { KubectlV28Layer } from '@aws-cdk/lambda-layer-kubectl-v28';
import { Stack, StackProps } from 'aws-cdk-lib';
import { InstanceClass, InstanceSize, InstanceType, IVpc, Vpc } from 'aws-cdk-lib/aws-ec2';
import { Cluster, KubernetesVersion } from 'aws-cdk-lib/aws-eks';
import { ManagedPolicy } from 'aws-cdk-lib/aws-iam';
import { Karpenter } from 'cdk-eks-karpenter';
import { Construct } from 'constructs';
export class NodeAutoscaling extends Construct {
constructor(
scope: Construct,
id: string,
{
cluster,
subnetIds,
}: {
cluster: Cluster;
subnetIds: string[];
},
) {
super(scope, id);
const karpenter = new Karpenter(this, 'Karpenter', {
cluster,
namespace: 'karpenter',
version: 'v0.34.1',
});
const nodeClass = karpenter.addEC2NodeClass('nodeclass', {
amiFamily: 'AL2',
subnetSelectorTerms: subnetIds.map((subnetId) => ({ id: subnetId })),
securityGroupSelectorTerms: [
{
tags: {
'aws:eks:cluster-name': cluster.clusterName,
},
},
],
role: karpenter.nodeRole.roleName,
});
karpenter.addNodePool('nodepool', {
template: {
spec: {
nodeClassRef: {
apiVersion: 'karpenter.k8s.aws/v1beta1',
kind: 'EC2NodeClass',
name: nodeClass.name,
},
requirements: [
{
key: 'karpenter.sh/capacity-type',
operator: 'In',
values: ['on-demand'],
},
{
key: 'karpenter.k8s.aws/instance-category',
operator: 'In',
values: ['m'],
},
{
key: 'karpenter.k8s.aws/instance-generation',
operator: 'In',
values: ['5', '6', '7'],
},
{
key: 'kubernetes.io/arch',
operator: 'In',
values: ['amd64'],
},
],
},
},
});
karpenter.addManagedPolicyToKarpenterRole(
ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'),
);
}
}
export class EksCluster extends Construct {
public readonly cluster: Cluster;
constructor(
scope: Construct,
id: string,
{
environment,
instanceName,
vpc,
}: {
environment: string;
instanceName: string;
vpc: IVpc;
},
) {
super(scope, id);
const kubectlLayer = new KubectlV28Layer(this, 'KubectlLayer');
this.cluster = new Cluster(this, 'Cluster', {
clusterName: `eks-example-${instanceName}-${environment}`,
defaultCapacity: 3,
defaultCapacityInstance: InstanceType.of(InstanceClass.M5, InstanceSize.LARGE),
kubectlLayer,
outputConfigCommand: true,
outputMastersRoleArn: true,
version: KubernetesVersion.V1_28,
vpc,
});
new NodeAutoscaling(this, 'NodeAutoscaling', {
cluster: this.cluster,
subnetIds: vpc.privateSubnets.map(({ subnetId }) => subnetId), // the landing zone creates the subnets in the following pattern --
});
}
}
export class MinBrokenEks extends Stack {
constructor(scope: Construct, id: string, props: StackProps) {
super(scope, id, props);
const vpc = Vpc.fromLookup(this, 'Vpc', { vpcName: vpcName(CONFIG.environment) });
this.configureClusterAndRoles({ vpc });
}
private configureClusterAndRoles({ vpc }: { vpc: IVpc }) {
const cluster = new EksCluster(this, 'EksCluster', {
environment: CONFIG.environment,
instanceName: CONFIG.instanceName,
vpc,
});
return cluster;
}
}
```
Contributor guide
Assessment
This issue has not been assessed yet.