From 2617678cef455c1241fdc6ca1241055069ba1ef9 Mon Sep 17 00:00:00 2001 From: Aurel Canciu Date: Sat, 13 Feb 2021 23:09:10 +0200 Subject: [PATCH] Support recreating objects on immutable field updates Allow passing --force to kubectl apply. Useful when dealing with immutable field changes in resources. Signed-off-by: Aurel Canciu --- .github/workflows/e2e.yaml | 2 + Makefile | 2 +- api/go.mod | 2 +- api/go.sum | 4 + api/v1beta1/kustomization_types.go | 14 +- ...mize.toolkit.fluxcd.io_kustomizations.yaml | 11 +- controllers/kustomization_controller.go | 15 +- controllers/kustomization_controller_test.go | 145 ++++++++++++++++++ docs/api/kustomize.md | 36 ++++- go.mod | 2 +- go.sum | 4 + 11 files changed, 224 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3cfade8c..993b5767 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -31,6 +31,8 @@ jobs: uses: fluxcd/pkg//actions/kubebuilder@main - name: Setup Kubectl uses: fluxcd/pkg/actions/kubectl@main + with: + version: 1.20.2 - name: Run tests run: make test env: diff --git a/Makefile b/Makefile index e78c2bb2..40d92e7b 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ ifeq (, $(shell which controller-gen)) CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ cd $$CONTROLLER_GEN_TMP_DIR ;\ go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } CONTROLLER_GEN=$(GOBIN)/controller-gen diff --git a/api/go.mod b/api/go.mod index 0ad7e429..b2f57366 100644 --- a/api/go.mod +++ b/api/go.mod @@ -8,5 +8,5 @@ require ( github.com/fluxcd/pkg/runtime v0.8.0 k8s.io/apiextensions-apiserver v0.20.2 k8s.io/apimachinery v0.20.2 - sigs.k8s.io/controller-runtime v0.8.0 + sigs.k8s.io/controller-runtime v0.8.2 ) diff --git a/api/go.sum b/api/go.sum index 6d29afe2..40155086 100644 --- a/api/go.sum +++ b/api/go.sum @@ -658,12 +658,16 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc= sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= +sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM= +sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go index ab443f7c..55ec4948 100644 --- a/api/v1beta1/kustomization_types.go +++ b/api/v1beta1/kustomization_types.go @@ -17,9 +17,10 @@ limitations under the License. package v1beta1 import ( + "time" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" - "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -123,10 +124,19 @@ type KustomizationSpec struct { Timeout *metav1.Duration `json:"timeout,omitempty"` // Validate the Kubernetes objects before applying them on the cluster. - // The validation strategy can be 'client' (local dry-run), 'server' (APIServer dry-run) or 'none'. + // The validation strategy can be 'client' (local dry-run), 'server' + // (APIServer dry-run) or 'none'. + // When 'Force' is 'true', validation will fallback to 'client' if set to + // 'server' because server-side validation is not supported in this scenario. // +kubebuilder:validation:Enum=none;client;server // +optional Validation string `json:"validation,omitempty"` + + // Force instructs the controller to recreate resources in the situation + // when dealing with immutable field changes. + // +kubebuilder:default:=false + // +optional + Force bool `json:"force,omitempty"` } // Decryption defines how decryption is handled for Kubernetes manifests. diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index 1918ac12..003cdc88 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: kustomizations.kustomize.toolkit.fluxcd.io spec: @@ -88,6 +88,11 @@ spec: - name type: object type: array + force: + default: false + description: Force instructs the controller to recreate resources + in the situation when dealing with immutable field changes. + type: boolean healthChecks: description: A list of resources to be included in the health assessment. items: @@ -318,7 +323,9 @@ spec: validation: description: Validate the Kubernetes objects before applying them on the cluster. The validation strategy can be 'client' (local dry-run), - 'server' (APIServer dry-run) or 'none'. + 'server' (APIServer dry-run) or 'none'. When 'Force' is 'true', + validation will fallback to 'client' if set to 'server' because + server-side validation is not supported in this scenario. enum: - none - client diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 94538368..8d65e62a 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -550,8 +550,15 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku applyCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp", - dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), kustomization.Spec.Validation) + validation := kustomization.Spec.Validation + if validation == "server" && kustomization.Spec.Force { + // Use client-side validation with force + validation = "client" + (logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled")) + } + + cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t", + dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), validation, kustomization.Spec.Force) if kustomization.Spec.KubeConfig != nil { kubeConfig, err := imp.WriteKubeConfig(ctx) @@ -589,8 +596,8 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto defer cancel() fieldManager := "kustomize-controller" - cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp", - dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String()) + cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp --force=%t", + dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String(), kustomization.Spec.Force) if kustomization.Spec.KubeConfig != nil { kubeConfig, err := imp.WriteKubeConfig(ctx) diff --git a/controllers/kustomization_controller_test.go b/controllers/kustomization_controller_test.go index 561a698c..c4635c64 100644 --- a/controllers/kustomization_controller_test.go +++ b/controllers/kustomization_controller_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -171,6 +172,7 @@ var _ = Describe("KustomizationReconciler", func() { Suspend: false, Timeout: nil, Validation: "client", + Force: false, PostBuild: &kustomizev1.PostBuild{ Substitute: map[string]string{"region": "eu-central-1"}, }, @@ -244,5 +246,148 @@ metadata: expectRevision: "branch/commit1", }), ) + + Describe("Kustomization resource replacement", func() { + deploymentManifest := func(namespace, selector string) string { + return fmt.Sprintf(`--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test + namespace: %s +spec: + selector: + matchLabels: + app: %[2]s + template: + metadata: + labels: + app: %[2]s + spec: + containers: + - name: test + image: podinfo +`, + namespace, selector) + } + + It("should replace immutable field resource using force", func() { + manifests := []testserver.File{ + { + Name: "deployment.yaml", + Body: deploymentManifest(namespace.Name, "v1"), + }, + } + artifact, err := httpServer.ArtifactFromFiles(manifests) + Expect(err).NotTo(HaveOccurred()) + + url := fmt.Sprintf("%s/%s", httpServer.URL(), artifact) + + repositoryName := types.NamespacedName{ + Name: fmt.Sprintf("%s", randStringRunes(5)), + Namespace: namespace.Name, + } + repository := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: repositoryName.Name, + Namespace: repositoryName.Namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: "https://github.com/test/repository", + Interval: metav1.Duration{Duration: reconciliationInterval}, + }, + Status: sourcev1.GitRepositoryStatus{ + Conditions: []metav1.Condition{ + { + Type: meta.ReadyCondition, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: sourcev1.GitOperationSucceedReason, + }, + }, + URL: url, + Artifact: &sourcev1.Artifact{ + Path: url, + URL: url, + Revision: "v1", + LastUpdateTime: metav1.Now(), + }, + }, + } + Expect(k8sClient.Create(context.Background(), repository)).To(Succeed()) + Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed()) + defer k8sClient.Delete(context.Background(), repository) + + kName := types.NamespacedName{ + Name: fmt.Sprintf("%s", randStringRunes(5)), + Namespace: namespace.Name, + } + k := &kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: kName.Name, + Namespace: kName.Namespace, + }, + Spec: kustomizev1.KustomizationSpec{ + KubeConfig: kubeconfig, + Interval: metav1.Duration{Duration: reconciliationInterval}, + Path: "./", + Prune: true, + SourceRef: kustomizev1.CrossNamespaceSourceReference{ + Kind: sourcev1.GitRepositoryKind, + Name: repository.Name, + }, + Suspend: false, + Timeout: &metav1.Duration{Duration: 60 * time.Second}, + Validation: "client", + Force: true, + }, + } + Expect(k8sClient.Create(context.Background(), k)).To(Succeed()) + defer k8sClient.Delete(context.Background(), k) + + got := &kustomizev1.Kustomization{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), kName, got) + c := apimeta.FindStatusCondition(got.Status.Conditions, meta.ReadyCondition) + return c != nil && c.Reason == meta.ReconciliationSucceededReason + }, timeout, interval).Should(BeTrue()) + Expect(got.Status.LastAppliedRevision).To(Equal("v1")) + + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed()) + Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v1")) + + manifests = []testserver.File{ + { + Name: "deployment.yaml", + Body: deploymentManifest(namespace.Name, "v2"), + }, + } + + artifact, err = httpServer.ArtifactFromFiles(manifests) + Expect(err).NotTo(HaveOccurred()) + + url = fmt.Sprintf("%s/%s", httpServer.URL(), artifact) + + repository.Status.URL = url + repository.Status.Artifact = &sourcev1.Artifact{ + Path: url, + URL: url, + Revision: "v2", + LastUpdateTime: metav1.Now(), + } + Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed()) + + lastAppliedRev := got.Status.LastAppliedRevision + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), kName, got) + return apimeta.IsStatusConditionTrue(got.Status.Conditions, meta.ReadyCondition) && got.Status.LastAppliedRevision != lastAppliedRev + }, timeout, interval).Should(BeTrue()) + Expect(got.Status.LastAppliedRevision).To(Equal("v2")) + + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed()) + Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v2")) + }) + }) }) }) diff --git a/docs/api/kustomize.md b/docs/api/kustomize.md index 83eb1617..5461b4a2 100644 --- a/docs/api/kustomize.md +++ b/docs/api/kustomize.md @@ -317,7 +317,23 @@ string (Optional)

Validate the Kubernetes objects before applying them on the cluster. -The validation strategy can be ‘client’ (local dry-run), ‘server’ (APIServer dry-run) or ‘none’.

+The validation strategy can be ‘client’ (local dry-run), ‘server’ +(APIServer dry-run) or ‘none’. +When ‘Force’ is ‘true’, validation will fallback to ‘client’ if set to +‘server’ because server-side validation is not supported in this scenario.

+ + + + +force
+ +bool + + + +(Optional) +

Force instructs the controller to recreate resources in the situation +when dealing with immutable field changes.

@@ -760,7 +776,23 @@ string (Optional)

Validate the Kubernetes objects before applying them on the cluster. -The validation strategy can be ‘client’ (local dry-run), ‘server’ (APIServer dry-run) or ‘none’.

+The validation strategy can be ‘client’ (local dry-run), ‘server’ +(APIServer dry-run) or ‘none’. +When ‘Force’ is ‘true’, validation will fallback to ‘client’ if set to +‘server’ because server-side validation is not supported in this scenario.

+ + + + +force
+ +bool + + + +(Optional) +

Force instructs the controller to recreate resources in the situation +when dealing with immutable field changes.

diff --git a/go.mod b/go.mod index fab8c972..8301e058 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( k8s.io/cli-runtime v0.20.2 // indirect k8s.io/client-go v0.20.2 sigs.k8s.io/cli-utils v0.20.2 - sigs.k8s.io/controller-runtime v0.8.0 + sigs.k8s.io/controller-runtime v0.8.2 sigs.k8s.io/kustomize/api v0.7.4 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 36b6374f..2348d1ec 100644 --- a/go.sum +++ b/go.sum @@ -1217,6 +1217,8 @@ k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -1234,6 +1236,8 @@ sigs.k8s.io/cli-utils v0.20.2/go.mod h1:uT5cVxMrOoRplL8umtrJx5R51ZWsIKD7lQfPtot8 sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc= sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= +sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM= +sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.7.4 h1:WyuZ7ZI7U978udBWMFdKlxjTMOmF7BjpQA1BuygyArY=