Support recreating objects on immutable field updates

Allow passing --force to kubectl apply. Useful when dealing with
immutable field changes in resources.

Signed-off-by: Aurel Canciu <aurelcanciu@gmail.com>

.github/workflows/e2e.yaml

@@ -31,6 +31,8 @@ jobs:
         uses: fluxcd/pkg//actions/kubebuilder@main
       - name: Setup Kubectl
         uses: fluxcd/pkg/actions/kubectl@main
+        with:
+          version: 1.20.2
       - name: Run tests
         run: make test
         env:

Makefile

@@ -107,7 +107,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

api/go.mod

@@ -8,5 +8,5 @@ require (
 	github.com/fluxcd/pkg/runtime v0.8.0
 	k8s.io/apiextensions-apiserver v0.20.2
 	k8s.io/apimachinery v0.20.2
-	sigs.k8s.io/controller-runtime v0.8.0
+	sigs.k8s.io/controller-runtime v0.8.2
 )

api/go.sum

@@ -658,12 +658,16 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc=
 sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4=
+sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM=
+sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/v1beta1/kustomization_types.go

@@ -17,9 +17,10 @@ limitations under the License.
 package v1beta1
 
 import (
+	"time"
+
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
-	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -124,10 +125,19 @@ type KustomizationSpec struct {
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
 	// Validate the Kubernetes objects before applying them on the cluster.
-	// The validation strategy can be 'client' (local dry-run), 'server' (APIServer dry-run) or 'none'.
+	// The validation strategy can be 'client' (local dry-run), 'server'
+	// (APIServer dry-run) or 'none'.
+	// When 'Force' is 'true', validation will fallback to 'client' if set to
+	// 'server' because server-side validation is not supported in this scenario.
 	// +kubebuilder:validation:Enum=none;client;server
 	// +optional
 	Validation string `json:"validation,omitempty"`
+
+	// Force instructs the controller to recreate resources in the situation
+	// when dealing with immutable field changes.
+	// +kubebuilder:default:=false
+	// +optional
+	Force bool `json:"force,omitempty"`
 }
 
 // Decryption defines how decryption is handled for Kubernetes manifests.

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.3.0
+    controller-gen.kubebuilder.io/version: v0.4.1
   creationTimestamp: null
   name: kustomizations.kustomize.toolkit.fluxcd.io
 spec:
@@ -88,6 +88,11 @@ spec:
                   - name
                   type: object
                 type: array
+              force:
+                default: false
+                description: Force instructs the controller to recreate resources
+                  in the situation when dealing with immutable field changes.
+                type: boolean
               healthChecks:
                 description: A list of resources to be included in the health assessment.
                 items:
@@ -346,7 +351,9 @@ spec:
               validation:
                 description: Validate the Kubernetes objects before applying them
                   on the cluster. The validation strategy can be 'client' (local dry-run),
-                  'server' (APIServer dry-run) or 'none'.
+                  'server' (APIServer dry-run) or 'none'. When 'Force' is 'true',
+                  validation will fallback to 'client' if set to 'server' because
+                  server-side validation is not supported in this scenario.
                 enum:
                 - none
                 - client

controllers/kustomization_controller.go

@@ -571,8 +571,15 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
 	applyCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
-	cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp",
-		dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), kustomization.Spec.Validation)
+	validation := kustomization.Spec.Validation
+	if validation == "server" && kustomization.Spec.Force {
+		// Use client-side validation with force
+		validation = "client"
+		(logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
+	}
+
+	cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t",
+		dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), validation, kustomization.Spec.Force)
 
 	if kustomization.Spec.KubeConfig != nil {
 		kubeConfig, err := imp.WriteKubeConfig(ctx)
@@ -610,8 +617,8 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
 	defer cancel()
 	fieldManager := "kustomize-controller"
 
-	cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp",
-		dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String())
+	cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp --force=%t",
+		dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String(), kustomization.Spec.Force)
 
 	if kustomization.Spec.KubeConfig != nil {
 		kubeConfig, err := imp.WriteKubeConfig(ctx)

controllers/kustomization_controller_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -197,6 +198,7 @@ var _ = Describe("KustomizationReconciler", func() {
 					Suspend:    false,
 					Timeout:    nil,
 					Validation: "client",
+					Force:      false,
 					PostBuild: &kustomizev1.PostBuild{
 						Substitute: map[string]string{"region": "eu-central-1"},
 						SubstituteFrom: []kustomizev1.SubstituteReference{
@@ -282,5 +284,148 @@ metadata:
 				expectRevision: "branch/commit1",
 			}),
 		)
+
+		Describe("Kustomization resource replacement", func() {
+			deploymentManifest := func(namespace, selector string) string {
+				return fmt.Sprintf(`---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test
+  namespace: %s
+spec:
+  selector:
+    matchLabels:
+      app: %[2]s
+  template:
+    metadata:
+      labels:
+        app: %[2]s
+    spec:
+      containers:
+      - name: test
+        image: podinfo
+`,
+					namespace, selector)
+			}
+
+			It("should replace immutable field resource using force", func() {
+				manifests := []testserver.File{
+					{
+						Name: "deployment.yaml",
+						Body: deploymentManifest(namespace.Name, "v1"),
+					},
+				}
+				artifact, err := httpServer.ArtifactFromFiles(manifests)
+				Expect(err).NotTo(HaveOccurred())
+
+				url := fmt.Sprintf("%s/%s", httpServer.URL(), artifact)
+
+				repositoryName := types.NamespacedName{
+					Name:      fmt.Sprintf("%s", randStringRunes(5)),
+					Namespace: namespace.Name,
+				}
+				repository := &sourcev1.GitRepository{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      repositoryName.Name,
+						Namespace: repositoryName.Namespace,
+					},
+					Spec: sourcev1.GitRepositorySpec{
+						URL:      "https://github.com/test/repository",
+						Interval: metav1.Duration{Duration: reconciliationInterval},
+					},
+					Status: sourcev1.GitRepositoryStatus{
+						Conditions: []metav1.Condition{
+							{
+								Type:               meta.ReadyCondition,
+								Status:             metav1.ConditionTrue,
+								LastTransitionTime: metav1.Now(),
+								Reason:             sourcev1.GitOperationSucceedReason,
+							},
+						},
+						URL: url,
+						Artifact: &sourcev1.Artifact{
+							Path:           url,
+							URL:            url,
+							Revision:       "v1",
+							LastUpdateTime: metav1.Now(),
+						},
+					},
+				}
+				Expect(k8sClient.Create(context.Background(), repository)).To(Succeed())
+				Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed())
+				defer k8sClient.Delete(context.Background(), repository)
+
+				kName := types.NamespacedName{
+					Name:      fmt.Sprintf("%s", randStringRunes(5)),
+					Namespace: namespace.Name,
+				}
+				k := &kustomizev1.Kustomization{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      kName.Name,
+						Namespace: kName.Namespace,
+					},
+					Spec: kustomizev1.KustomizationSpec{
+						KubeConfig: kubeconfig,
+						Interval:   metav1.Duration{Duration: reconciliationInterval},
+						Path:       "./",
+						Prune:      true,
+						SourceRef: kustomizev1.CrossNamespaceSourceReference{
+							Kind: sourcev1.GitRepositoryKind,
+							Name: repository.Name,
+						},
+						Suspend:    false,
+						Timeout:    &metav1.Duration{Duration: 60 * time.Second},
+						Validation: "client",
+						Force:      true,
+					},
+				}
+				Expect(k8sClient.Create(context.Background(), k)).To(Succeed())
+				defer k8sClient.Delete(context.Background(), k)
+
+				got := &kustomizev1.Kustomization{}
+				Eventually(func() bool {
+					_ = k8sClient.Get(context.Background(), kName, got)
+					c := apimeta.FindStatusCondition(got.Status.Conditions, meta.ReadyCondition)
+					return c != nil && c.Reason == meta.ReconciliationSucceededReason
+				}, timeout, interval).Should(BeTrue())
+				Expect(got.Status.LastAppliedRevision).To(Equal("v1"))
+
+				deployment := &appsv1.Deployment{}
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed())
+				Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v1"))
+
+				manifests = []testserver.File{
+					{
+						Name: "deployment.yaml",
+						Body: deploymentManifest(namespace.Name, "v2"),
+					},
+				}
+
+				artifact, err = httpServer.ArtifactFromFiles(manifests)
+				Expect(err).NotTo(HaveOccurred())
+
+				url = fmt.Sprintf("%s/%s", httpServer.URL(), artifact)
+
+				repository.Status.URL = url
+				repository.Status.Artifact = &sourcev1.Artifact{
+					Path:           url,
+					URL:            url,
+					Revision:       "v2",
+					LastUpdateTime: metav1.Now(),
+				}
+				Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed())
+
+				lastAppliedRev := got.Status.LastAppliedRevision
+				Eventually(func() bool {
+					_ = k8sClient.Get(context.Background(), kName, got)
+					return apimeta.IsStatusConditionTrue(got.Status.Conditions, meta.ReadyCondition) && got.Status.LastAppliedRevision != lastAppliedRev
+				}, timeout, interval).Should(BeTrue())
+				Expect(got.Status.LastAppliedRevision).To(Equal("v2"))
+
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed())
+				Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v2"))
+			})
+		})
 	})
 })

docs/api/kustomize.md

@@ -317,7 +317,23 @@ string
 <td>
 <em>(Optional)</em>
 <p>Validate the Kubernetes objects before applying them on the cluster.
-The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo; (APIServer dry-run) or &lsquo;none&rsquo;.</p>
+The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo;
+(APIServer dry-run) or &lsquo;none&rsquo;.
+When &lsquo;Force&rsquo; is &lsquo;true&rsquo;, validation will fallback to &lsquo;client&rsquo; if set to
+&lsquo;server&rsquo; because server-side validation is not supported in this scenario.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>force</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Force instructs the controller to recreate resources in the situation
+when dealing with immutable field changes.</p>
 </td>
 </tr>
 </table>
@@ -760,7 +776,23 @@ string
 <td>
 <em>(Optional)</em>
 <p>Validate the Kubernetes objects before applying them on the cluster.
-The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo; (APIServer dry-run) or &lsquo;none&rsquo;.</p>
+The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo;
+(APIServer dry-run) or &lsquo;none&rsquo;.
+When &lsquo;Force&rsquo; is &lsquo;true&rsquo;, validation will fallback to &lsquo;client&rsquo; if set to
+&lsquo;server&rsquo; because server-side validation is not supported in this scenario.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>force</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Force instructs the controller to recreate resources in the situation
+when dealing with immutable field changes.</p>
 </td>
 </tr>
 </tbody>

go.mod

@@ -29,7 +29,7 @@ require (
 	k8s.io/cli-runtime v0.20.2 // indirect
 	k8s.io/client-go v0.20.2
 	sigs.k8s.io/cli-utils v0.20.2
-	sigs.k8s.io/controller-runtime v0.8.0
+	sigs.k8s.io/controller-runtime v0.8.2
 	sigs.k8s.io/kustomize/api v0.7.4
 	sigs.k8s.io/yaml v1.2.0
 )

go.sum

@@ -1217,6 +1217,8 @@ k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
@@ -1234,6 +1236,8 @@ sigs.k8s.io/cli-utils v0.20.2/go.mod h1:uT5cVxMrOoRplL8umtrJx5R51ZWsIKD7lQfPtot8
 sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns=
 sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc=
 sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4=
+sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM=
+sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
 sigs.k8s.io/kustomize/api v0.7.4 h1:WyuZ7ZI7U978udBWMFdKlxjTMOmF7BjpQA1BuygyArY=