Skip to content

Commit

Permalink
Merge pull request #890 from paulmowat/feat/karpenter_nodetemplate_bl…
Browse files Browse the repository at this point in the history 
…ockdevicemapping

feat: adding support for karpenter blockdevicemapping
  • Loading branch information
@shapirov103
shapirov103 committed Dec 15, 2023
2 parents 1486357 + 87d2bb5 commit 425decc
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 39 deletions.
13 changes: 11 additions & 2 deletions docs/addons/karpenter.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Karpenter works by:
import 'source-map-support/register';
import * as cdk from 'aws-cdk-lib';
import * as blueprints from '@aws-quickstart/eks-blueprints';
import { EbsDeviceVolumeType } from 'aws-cdk-lib/aws-ec2';

const app = new cdk.App();

Expand Down Expand Up @@ -53,7 +54,15 @@ const karpenterAddOn = new blueprints.addons.KarpenterAddOn({
interruptionHandling: true,
tags: {
schedule: 'always-on'
}
},
blockDeviceMappings: [{
deviceName: "/dev/xvda",
ebs: {
volumeSize: 100,
volumeType: EbsDeviceVolumeType.GP3,
deleteOnTermination: true
},
}],
});

const blueprint = blueprints.EksBlueprint.builder()
Expand Down Expand Up @@ -86,7 +95,7 @@ blueprints-addon-karpenter-54fd978b89-hclmp 2/2 Running 0 99m
2. Creates `karpenter` namespace.
3. Creates Kubernetes Service Account, and associate AWS IAM Role with Karpenter Controller Policy attached using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html).
4. Deploys Karpenter helm chart in the `karpenter` namespace, configuring cluster name and cluster endpoint on the controller by default.
5. (Optionally) provisions a default Karpenter Provisioner and AWSNodeTemplate CRD based on user-provided parameters such as [spec.requirements](https://karpenter.sh/docs/concepts/nodepools/#spectemplatespecrequirements), [AMI type](https://karpenter.sh/docs/concepts/nodeclasses/#specamifamily),[weight](https://karpenter.sh/docs/concepts/provisioners/#specweight), [Subnet Selector](https://karpenter.sh/docs/concepts/nodeclasses/#specsubnetselectorterms), [Security Group Selector](https://karpenter.sh/docs/concepts/nodeclasses/#specsecuritygroupselectorterms) and [Tags](https://karpenter.sh/docs/concepts/nodeclasses/#spectags). If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.
5. (Optionally) provisions a default Karpenter Provisioner and AWSNodeTemplate CRD based on user-provided parameters such as [spec.requirements](https://karpenter.sh/docs/concepts/nodepools/#spectemplatespecrequirements), [AMI type](https://karpenter.sh/docs/concepts/nodeclasses/#specamifamily),[weight](https://karpenter.sh/docs/concepts/provisioners/#specweight), [Subnet Selector](https://karpenter.sh/docs/concepts/nodeclasses/#specsubnetselectorterms), [Security Group Selector](https://karpenter.sh/docs/concepts/nodeclasses/#specsecuritygroupselectorterms), [Tags](https://karpenter.sh/docs/concepts/nodeclasses/#spectags) and [BlockDeviceMappings](https://karpenter.sh/v0.30/concepts/node-templates/#specblockdevicemappings). If created, the provisioner will discover the EKS VPC subnets and security groups to launch the nodes with.

**NOTE:**
1. The default provisioner is created only if both the subnet tags and the security group tags are provided.
Expand Down
30 changes: 29 additions & 1 deletion lib/addons/karpenter/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,31 @@ import * as sqs from 'aws-cdk-lib/aws-sqs';
import { Rule } from 'aws-cdk-lib/aws-events';
import { SqsQueue } from 'aws-cdk-lib/aws-events-targets';
import { Cluster } from 'aws-cdk-lib/aws-eks';
import { EbsDeviceVolumeType } from 'aws-cdk-lib/aws-ec2';

export interface BlockDeviceMapping {
deviceName?: string;
virtualName?: string;
ebs?: EbsVolumeMapping;
noDevice?: string;
}

export interface EbsVolumeMapping {
deleteOnTermination?: boolean;
iops?: number;
snapshotId?: string;
volumeSize?: number;
volumeType?: EbsDeviceVolumeType;
kmsKeyId?: string;
throughput?: number;
outpostArn?: string;
encrypted?: boolean;
}

/**
* Configuration options for the add-on
*/
interface KarpenterAddOnProps extends HelmAddOnUserProps {
export interface KarpenterAddOnProps extends HelmAddOnUserProps {
/**
* Taints for the provisioned nodes - Taints may prevent pods from scheduling if they are not tolerated by the pod.
*/
Expand Down Expand Up @@ -131,6 +150,13 @@ interface KarpenterAddOnProps extends HelmAddOnUserProps {
* This ensures that Karpenter is able to correctly auto-discover machines that it owns.
*/
tags?: Values;

/**
* BlockDeviceMappings allows you to specify the block device mappings for the instances.
* This is a list of mappings, where each mapping consists of a device name and an EBS configuration.
* If you leave this blank, it will use the Karpenter default.
*/
blockDeviceMappings?: BlockDeviceMapping[];
}

const KARPENTER = 'karpenter';
Expand Down Expand Up @@ -190,6 +216,7 @@ export class KarpenterAddOn extends HelmAddOn {
const interruption = this.options.interruptionHandling || false;
const limits = this.options.limits || null;
const tags = this.options.tags || null;
const blockDeviceMappings = this.options.blockDeviceMappings || [];

// Various checks for version errors
const consolidation = this.versionFeatureChecksForError(clusterInfo, version, weight, consol, repo, ttlSecondsAfterEmpty, interruption);
Expand Down Expand Up @@ -341,6 +368,7 @@ export class KarpenterAddOn extends HelmAddOn {
subnetSelector: subnetTags,
securityGroupSelector: sgTags,
tags: tags,
blockDeviceMappings: blockDeviceMappings
},
});
nodeTemplate.node.addDependency(provisioner);
Expand Down
126 changes: 90 additions & 36 deletions test/karpenter.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import * as cdk from 'aws-cdk-lib';
import * as blueprints from '../lib';
import { Template } from 'aws-cdk-lib/assertions';
import { EbsDeviceVolumeType } from 'aws-cdk-lib/aws-ec2';
import { BlockDeviceMapping, EbsVolumeMapping } from "../lib";

describe('Unit tests for Karpenter addon', () => {

Expand Down Expand Up @@ -114,42 +116,94 @@ describe('Unit tests for Karpenter addon', () => {
});
}).toThrow("Template has 0 resources with type AWS::SQS::Queue.");
});
test("Stack creation succeeds with custom values overrides", async () => {
const app = new cdk.App();

const blueprint = blueprints.EksBlueprint.builder();

const stack = await blueprint
.version("auto")
.account("123567891")
.region("us-west-1")
.addOns(
new blueprints.KarpenterAddOn({
version: 'v0.29.2',
values: {
settings: {
aws: {
enableENILimitedPodDensity: true,
interruptionQueueName: "override-queue-name",
},

test("Stack creation succeeds with custom values overrides", async () => {
const app = new cdk.App();

const blueprint = blueprints.EksBlueprint.builder();

const stack = await blueprint
.version("auto")
.account("123567891")
.region("us-west-1")
.addOns(
new blueprints.KarpenterAddOn({
version: 'v0.29.2',
values: {
settings: {
aws: {
enableENILimitedPodDensity: true,
interruptionQueueName: "override-queue-name",
},
},
},
},
})
)
.buildAsync(app, "stack-with-values-overrides");

const template = Template.fromStack(stack);
template.hasResourceProperties("Custom::AWSCDK-EKS-HelmChart", {
Chart: "karpenter",
})
)
.buildAsync(app, "stack-with-values-overrides");

const template = Template.fromStack(stack);
template.hasResourceProperties("Custom::AWSCDK-EKS-HelmChart", {
Chart: "karpenter",
});
const karpenter = template.findResources("Custom::AWSCDK-EKS-HelmChart");
const properties = Object.values(karpenter).pop();
const values = properties?.Properties?.Values;
expect(values).toBeDefined();
const valuesStr = JSON.stringify(values);
expect(valuesStr).toContain("defaultInstanceProfile");
expect(valuesStr).toContain("override-queue-name");
expect(valuesStr).toContain("enableENILimitedPodDensity");
});
const karpenter = template.findResources("Custom::AWSCDK-EKS-HelmChart");
const properties = Object.values(karpenter).pop();
const values = properties?.Properties?.Values;
expect(values).toBeDefined();
const valuesStr = JSON.stringify(values);
expect(valuesStr).toContain("defaultInstanceProfile");
expect(valuesStr).toContain("override-queue-name");
expect(valuesStr).toContain("enableENILimitedPodDensity");
});
});

test("Stack creation succeeds with custom values overrides for blockDeviceMapping", async () => {
const app = new cdk.App();

const blueprint = blueprints.EksBlueprint.builder();

const ebsVolumeMapping: EbsVolumeMapping = {
volumeSize: 20,
volumeType: EbsDeviceVolumeType.GP3,
deleteOnTermination: true,
};

const blockDeviceMapping: BlockDeviceMapping = {
deviceName: "/dev/xvda",
ebs: ebsVolumeMapping,
};

const stack = await blueprint
.version("auto")
.account("123567891")
.region("us-west-1")
.addOns(
new blueprints.KarpenterAddOn({
version: 'v0.29.2',
subnetTags: {
"Name": "blueprint-construct-dev/blueprint-construct-dev-vpc/PrivateSubnet1",
},
securityGroupTags: {
"kubernetes.io/cluster/blueprint-construct-dev": "owned",
},
blockDeviceMappings: [blockDeviceMapping]
})
)
.buildAsync(app, "stack-with-values-overrides-blockdevicemapping");

const template = Template.fromStack(stack);
const karpenterResources = template.findResources("Custom::AWSCDK-EKS-KubernetesResource");
const nodeTemplate = Object.values(karpenterResources).find((karpenterResource) => {
if (karpenterResource?.Properties?.Manifest) {
const manifest = karpenterResource.Properties.Manifest;
if (typeof manifest === "string" && manifest.includes('"kind":"AWSNodeTemplate"')) {
return true;
}
}
return false;
});
const manifest = JSON.parse(nodeTemplate?.Properties?.Manifest)[0];
expect(manifest.kind).toEqual('AWSNodeTemplate');
expect(manifest.spec.blockDeviceMappings).toBeDefined();
expect(manifest.spec.blockDeviceMappings.length).toEqual(1);
expect(manifest.spec.blockDeviceMappings[0]).toMatchObject(blockDeviceMapping);
});
});

0 comments on commit 425decc

Please sign in to comment.