Merge pull request #3374 from charlie-haley/feat/aws-node-env-vars

feat: add custom environment variables to the 'aws-node' DaemonSet

config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml

@@ -2175,6 +2175,122 @@ spec:
             description: AWSManagedControlPlaneSpec defines the desired state of an
               Amazon EKS Cluster.
             properties:
+              VpcCni:
+                description: VpcCni is used to set configuration options for the VPC
+                  CNI plugin
+                properties:
+                  env:
+                    description: Env defines a list of environment variables to apply
+                      to the `aws-node` DaemonSet
+                    items:
+                      description: EnvVar represents an environment variable present
+                        in a Container.
+                      properties:
+                        name:
+                          description: Name of the environment variable. Must be a
+                            C_IDENTIFIER.
+                          type: string
+                        value:
+                          description: 'Variable references $(VAR_NAME) are expanded
+                            using the previously defined environment variables in
+                            the container and any service environment variables. If
+                            a variable cannot be resolved, the reference in the input
+                            string will be unchanged. Double $$ are reduced to a single
+                            $, which allows for escaping the $(VAR_NAME) syntax: i.e.
+                            "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
+                            Escaped references will never be expanded, regardless
+                            of whether the variable exists or not. Defaults to "".'
+                          type: string
+                        valueFrom:
+                          description: Source for the environment variable's value.
+                            Cannot be used if value is not empty.
+                          properties:
+                            configMapKeyRef:
+                              description: Selects a key of a ConfigMap.
+                              properties:
+                                key:
+                                  description: The key to select.
+                                  type: string
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    TODO: Add other useful fields. apiVersion, kind,
+                                    uid?'
+                                  type: string
+                                optional:
+                                  description: Specify whether the ConfigMap or its
+                                    key must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                            fieldRef:
+                              description: 'Selects a field of the pod: supports metadata.name,
+                                metadata.namespace, `metadata.labels[''<KEY>'']`,
+                                `metadata.annotations[''<KEY>'']`, spec.nodeName,
+                                spec.serviceAccountName, status.hostIP, status.podIP,
+                                status.podIPs.'
+                              properties:
+                                apiVersion:
+                                  description: Version of the schema the FieldPath
+                                    is written in terms of, defaults to "v1".
+                                  type: string
+                                fieldPath:
+                                  description: Path of the field to select in the
+                                    specified API version.
+                                  type: string
+                              required:
+                              - fieldPath
+                              type: object
+                            resourceFieldRef:
+                              description: 'Selects a resource of the container: only
+                                resources limits and requests (limits.cpu, limits.memory,
+                                limits.ephemeral-storage, requests.cpu, requests.memory
+                                and requests.ephemeral-storage) are currently supported.'
+                              properties:
+                                containerName:
+                                  description: 'Container name: required for volumes,
+                                    optional for env vars'
+                                  type: string
+                                divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  description: Specifies the output format of the
+                                    exposed resources, defaults to "1"
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                resource:
+                                  description: 'Required: resource to select'
+                                  type: string
+                              required:
+                              - resource
+                              type: object
+                            secretKeyRef:
+                              description: Selects a key of a secret in the pod's
+                                namespace
+                              properties:
+                                key:
+                                  description: The key of the secret to select from.  Must
+                                    be a valid secret key.
+                                  type: string
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    TODO: Add other useful fields. apiVersion, kind,
+                                    uid?'
+                                  type: string
+                                optional:
+                                  description: Specify whether the Secret or its key
+                                    must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                          type: object
+                      required:
+                      - name
+                      type: object
+                    type: array
+                type: object
               additionalTags:
                 additionalProperties:
                   type: string

controlplane/eks/api/v1alpha3/conversion.go

@@ -44,6 +44,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Status.Bastion = restored.Status.Bastion
 	dst.Spec.OIDCIdentityProviderConfig = restored.Spec.OIDCIdentityProviderConfig
 	dst.Spec.KubeProxy = restored.Spec.KubeProxy
+	dst.Spec.VpcCni = restored.Spec.VpcCni
 
 	return nil
 }

controlplane/eks/api/v1alpha4/conversion.go

@@ -41,6 +41,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	dst.Spec.KubeProxy = restored.Spec.KubeProxy
+	dst.Spec.VpcCni = restored.Spec.VpcCni
 
 	return nil
 }
@@ -74,10 +75,6 @@ func (r *AWSManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) error {
 	return Convert_v1beta1_AWSManagedControlPlaneList_To_v1alpha4_AWSManagedControlPlaneList(src, r, nil)
 }
 
-func Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in *v1beta1.AWSManagedControlPlaneSpec, out *AWSManagedControlPlaneSpec, scope apiconversion.Scope) error {
-	return autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in, out, scope)
-}
-
 // Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus is a conversion function.
 func Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus(in *infrav1alpha4.NetworkStatus, out *infrav1beta1.NetworkStatus, s apiconversion.Scope) error {
 	return infrav1alpha4.Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus(in, out, s)
@@ -127,3 +124,7 @@ func Convert_v1beta1_Instance_To_v1alpha4_Instance(in *infrav1beta1.Instance, ou
 func Convert_v1alpha4_Instance_To_v1beta1_Instance(in *infrav1alpha4.Instance, out *infrav1beta1.Instance, s apiconversion.Scope) error {
 	return infrav1alpha4.Convert_v1alpha4_Instance_To_v1beta1_Instance(in, out, s)
 }
+
+func Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in *v1beta1.AWSManagedControlPlaneSpec, out *AWSManagedControlPlaneSpec, scope apiconversion.Scope) error {
+	return autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in, out, scope)
+}

controlplane/eks/api/v1beta1/awsmanagedcontrolplane_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1beta1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
@@ -167,6 +168,10 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned
 	// +kubebuilder:default=false
 	DisableVPCCNI bool `json:"disableVPCCNI,omitempty"`
 
+	// VpcCni is used to set configuration options for the VPC CNI plugin
+	// +optional
+	VpcCni VpcCni `json:"VpcCni,omitempty"`
+
 	// KubeProxy defines managed attributes of the kube-proxy daemonset
 	KubeProxy KubeProxy `json:"kubeProxy,omitempty"`
 }
@@ -182,6 +187,13 @@ type KubeProxy struct {
 	Disable bool `json:"disable,omitempty"`
 }
 
+// VpcCni specifies configuration related to the VPC CNI.
+type VpcCni struct {
+	// Env defines a list of environment variables to apply to the `aws-node` DaemonSet
+	// +optional
+	Env []corev1.EnvVar `json:"env,omitempty"`
+}
+
 // EndpointAccess specifies how control plane endpoints are accessible.
 type EndpointAccess struct {
 	// Public controls whether control plane endpoints are publicly accessible

docs/book/src/topics/eks/pod-networking.md

@@ -7,6 +7,28 @@ When creating a EKS cluster the Amazon VPC CNI will be used by default for Pod N
 ## Using the VPC CNI Addon
 You can use an explicit version of the Amazon VPC CNI by using the **vpc-cni** EKS addon. See the [addons](./addons.md) documentation for further details of how to use addons.
 
+
+## Increase node pod limit
+You can increase the pod limit per-node as [per the upstream AWS documentation](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/). You'll need to enable the `vpc-cni` plugin addon on your EKS cluster as well as enable prefix assignment mode through the `ENABLE_PREFIX_DELEGATION` environment variable.
+
+```yaml
+kind: AWSManagedControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "capi-managed-test-control-plane"
+spec:
+  VpcCni:
+    env:
+    - name: ENABLE_PREFIX_DELEGATION
+      value: "true"
+  addons:
+  - name: vpc-cni
+    version: <replace_with_version>
+    conflictResolution: overwrite
+  associateOIDCProvider: true
+  disableVPCCNI: false
+```
+
 ## Using an alternative CNI
 
 There may be scenarios where you do not want to use the Amazon VPC CNI. EKS supports a number of alternative CNIs such as Calico, Cilium, and Weave Net (see [docs](https://docs.aws.amazon.com/eks/latest/userguide/alternate-cni-plugins.html) for full list).

pkg/cloud/scope/awsnode.go

@@ -20,6 +20,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
+	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud"
 )
 
@@ -37,4 +38,6 @@ type AWSNodeScope interface {
 	SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup
 	// DisableVPCCNI returns whether the AWS VPC CNI should be disabled
 	DisableVPCCNI() bool
+	// VpcCni specifies configuration related to the VPC CNI.
+	VpcCni() ekscontrolplanev1.VpcCni
 }

pkg/cloud/scope/managedcontrolplane.go

@@ -372,6 +372,11 @@ func (s *ManagedControlPlaneScope) DisableVPCCNI() bool {
 	return s.ControlPlane.Spec.DisableVPCCNI
 }
 
+// VpcCni returns a list of environment variables to apply to the `aws-node` DaemonSet.
+func (s *ManagedControlPlaneScope) VpcCni() ekscontrolplanev1.VpcCni {
+	return s.ControlPlane.Spec.VpcCni
+}
+
 func (s *ManagedControlPlaneScope) OIDCIdentityProviderConfig() *ekscontrolplanev1.OIDCIdentityProviderConfig {
 	return s.ControlPlane.Spec.OIDCIdentityProviderConfig
 }

pkg/cloud/services/awsnode/cni.go

@@ -55,10 +55,6 @@ func (s *Service) ReconcileCNI(ctx context.Context) error {
 		}
 	}
 
-	if s.scope.SecondaryCidrBlock() == nil {
-		return nil
-	}
-
 	var ds appsv1.DaemonSet
 	if err := remoteClient.Get(ctx, types.NamespacedName{Namespace: awsNodeNamespace, Name: awsNodeName}, &ds); err != nil {
 		if !apierrors.IsNotFound(err) {
@@ -67,6 +63,40 @@ func (s *Service) ReconcileCNI(ctx context.Context) error {
 		return ErrCNIMissing
 	}
 
+	envVars := s.scope.VpcCni().Env
+	if len(envVars) > 0 {
+		s.scope.Info("updating aws-node daemonset environment variables", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace())
+
+		for i := range ds.Spec.Template.Spec.Containers {
+			container := &ds.Spec.Template.Spec.Containers[i]
+			if container.Name == "aws-node" {
+				existingVars := s.filterEnv(container.Env)
+
+				for ei, e := range existingVars {
+					for ai, a := range envVars {
+						if e.Name == a.Name {
+							existingVars[ei].Value = a.Value
+
+							envVars = append(envVars[:ai], envVars[ai+1:]...)
+							break
+						}
+					}
+				}
+				container.Env = append(existingVars, envVars...)
+				break
+			}
+		}
+	}
+
+	if s.scope.SecondaryCidrBlock() == nil {
+		if len(envVars) > 0 {
+			if err = remoteClient.Update(ctx, &ds, &client.UpdateOptions{}); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
 	sgs, err := s.getSecurityGroups()
 	if err != nil {
 		return err

test/e2e/data/eks/cluster-template-eks-control-plane-only-withaddon.yaml

@@ -24,6 +24,12 @@ spec:
   region: "${AWS_REGION}"
   sshKeyName: "${AWS_SSH_KEY_NAME}"
   version: "${KUBERNETES_VERSION}"
+  VpcCni:
+    env:
+      - name: FOO
+        value: BAR
+      - name: ENABLE_PREFIX_DELEGATION
+        value: "true"
   addons:
     - name: "vpc-cni"
       version: "${VPC_ADDON_VERSION}"