From 14417267d64826b94d02b2685e9da8b07a708187 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sun, 28 Jan 2024 11:37:40 +0000 Subject: [PATCH 1/8] Create cluster-id file on MicroShift startup --- pkg/cmd/run.go | 1 + pkg/controllers/clusterid.go | 88 ++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 pkg/controllers/clusterid.go diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 17de1b8b0d1..0cf9afea7de 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -212,6 +212,7 @@ func RunMicroshift(cfg *config.Config) error { util.Must(m.AddService(node.NewKubeletServer(cfg))) util.Must(m.AddService(loadbalancerservice.NewLoadbalancerServiceController(cfg))) util.Must(m.AddService(controllers.NewKubeStorageVersionMigrator(cfg))) + util.Must(m.AddService(controllers.NewClusterID(cfg))) // Storing and clearing the env, so other components don't send the READY=1 until MicroShift is fully ready notifySocket := os.Getenv("NOTIFY_SOCKET") diff --git a/pkg/controllers/clusterid.go b/pkg/controllers/clusterid.go new file mode 100644 index 00000000000..09b4f192982 --- /dev/null +++ b/pkg/controllers/clusterid.go @@ -0,0 +1,88 @@ +/* +Copyright © 2024 MicroShift Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package controllers + +import ( + "context" + "os" + "path/filepath" + + "github.com/openshift/microshift/pkg/config" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" +) + +type ClusterID struct { + cfg *config.Config +} + +func NewClusterID(cfg *config.Config) *ClusterID { + s := &ClusterID{} + s.cfg = cfg + return s +} + +func (s *ClusterID) Name() string { return "cluster-id-manager" } +func (s *ClusterID) Dependencies() []string { + return []string{"kube-apiserver"} +} + +func (s *ClusterID) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { + defer close(stopped) + defer close(ready) + + // Read the 'kube-system' namespace attributes + restConfig, err := clientcmd.BuildConfigFromFlags("", s.cfg.KubeConfigPath(config.KubeAdmin)) + if err != nil { + panic(err) + } + coreClient := clientv1.NewForConfigOrDie(rest.AddUserAgent(restConfig, "core-agent")) + namespace, err := coreClient.Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}) + if err != nil { + panic(err) + } + + // Use the 'kube-system' namespace metadata UID as the MicroShift Cluster ID + clusterID := string(namespace.ObjectMeta.UID) + // Write /cluster-id file if it does not already exist + initClusterIDFile(clusterID) + // Log the cluster ID + klog.Infof("MicroShift Cluster ID: %v", clusterID) + + return ctx.Err() +} + +func initClusterIDFile(clusterID string) { + // The location of the cluster ID file + fileName := filepath.Join(config.DataDir, "cluster-id") + + // Do not create the cluster ID file if it already exists + _, err := os.Stat(fileName) + if !os.IsNotExist(err) { + return + } + + // Write the cluster ID to a new file + klog.Infof("Writing MicroShift Cluster ID '%v' to '%v'", clusterID, fileName) + err = os.WriteFile(fileName, []byte(clusterID), 0400) + if err != nil { + panic(err) + } +} From c6852dd3d3760642004198db28ee2288fe4d8423 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sun, 28 Jan 2024 11:47:26 +0000 Subject: [PATCH 2/8] Add cluster-id mention to debugging tips --- docs/user/debugging_tips.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/user/debugging_tips.md b/docs/user/debugging_tips.md index e7099e1667b..de460fe10b3 100644 --- a/docs/user/debugging_tips.md +++ b/docs/user/debugging_tips.md @@ -30,10 +30,27 @@ metadata: ## Checking the LVMS Version -Like the MicroShift version, the LVM version is available via a configmap. To get the version, run: +Like the MicroShift version, the LVM version is available via a configmap. +To get the version, run the following command. ```bash -$ oc get configmap -n kube-public lvms-version -ojsonpath='{.data.version}' +$ oc get configmap -n kube-public lvms-version -o jsonpath='{.data.version}' +``` + +## Checking the MicroShift Cluster ID + +MicroShift uses the `kube-system` namespace metadata UID value as a unique cluster identifier. +To get the cluster ID, run the following command. + +```bash +$ oc get namespaces kube-system -o jsonpath='{.metadata.uid}' +``` + +If the cluster is not running, examine the `/var/lib/microshift/cluster-id` file contents to +get the cluster ID. + +```bash +$ sudo cat /var/lib/microshift/cluster-id ``` ## Generating an SOS Report From 3ce76e0d481a6ed101eee79d9d8c9f4a1590c88e Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sun, 28 Jan 2024 15:34:45 +0200 Subject: [PATCH 3/8] Add the clusterid.robot e2e test --- test/suites/osconfig/clusterid.robot | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/suites/osconfig/clusterid.robot diff --git a/test/suites/osconfig/clusterid.robot b/test/suites/osconfig/clusterid.robot new file mode 100644 index 00000000000..9048e95e95c --- /dev/null +++ b/test/suites/osconfig/clusterid.robot @@ -0,0 +1,80 @@ +*** Settings *** +Documentation Tests verifying MicroShift cluster ID functionality + +Resource ../../resources/microshift-host.resource +Resource ../../resources/microshift-process.resource +Resource ../../resources/oc.resource +Resource ../../resources/ostree-health.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags restart slow + + +*** Variables *** +${CLUSTERID_FILE} /var/lib/microshift/cluster-id +${CLUSTERID_NS} kube-system + + +*** Test Cases *** +Compare Cluster ID From Namespace And File + [Documentation] Verify that cluster ID is the same when read + ... from namespace and file + ${name_id}= Get MicroShift Cluster ID From Namespace + ${file_id}= Get MicroShift Cluster ID From File + Should Be Equal As Strings ${name_id} ${file_id} + +Verify Cluster ID Change For New Database + [Documentation] Verify that cluster ID changes after MicroShift + ... database is cleaned and service restarted + + ${old_nid}= Get MicroShift Cluster ID From Namespace + ${old_fid}= Get MicroShift Cluster ID From File + Create New MicroShift Cluster + ${new_nid}= Get MicroShift Cluster ID From Namespace + ${new_fid}= Get MicroShift Cluster ID From File + + Should Be Equal As Strings ${old_nid} ${old_fid} + Should Be Equal As Strings ${new_nid} ${new_fid} + + Should Not Be Equal As Strings ${old_nid} ${new_nid} + Should Not Be Equal As Strings ${old_fid} ${new_fid} + + +*** Keywords *** +Setup + [Documentation] Set up all of the tests in this suite + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + +Teardown + [Documentation] Test suite teardown + Remove Kubeconfig + Logout MicroShift Host + +Create New MicroShift Cluster + [Documentation] Clean the database and restart MicroShift service. + Cleanup MicroShift --all --keep-images + Enable MicroShift + Start MicroShift + Setup Kubeconfig + Restart Greenboot And Wait For Success + +Get MicroShift Cluster ID From File + [Documentation] Read and return the cluster ID from the file. + ${stdout} ${rc}= Execute Command + ... cat ${CLUSTERID_FILE} + ... sudo=True return_rc=True return_stdout=True + Should Be Equal As Integers 0 ${rc} + + Should Not Be Empty ${stdout} + RETURN ${stdout} + +Get MicroShift Cluster ID From Namespace + [Documentation] Read and return the cluster ID from the kube-system namespace. + ${clusterid}= Oc Get Jsonpath namespaces ${CLUSTERID_NS} ${CLUSTERID_NS} .metadata.uid + + Should Not Be Empty ${clusterid} + RETURN ${clusterid} From 5b0e1f98c36e668f2ce92bc483617b48fce7838c Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Tue, 30 Jan 2024 06:33:53 +0000 Subject: [PATCH 4/8] Replace panic calls with error returns in the controller --- pkg/controllers/clusterid.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/clusterid.go b/pkg/controllers/clusterid.go index 09b4f192982..c8d1b789c95 100644 --- a/pkg/controllers/clusterid.go +++ b/pkg/controllers/clusterid.go @@ -51,38 +51,38 @@ func (s *ClusterID) Run(ctx context.Context, ready chan<- struct{}, stopped chan // Read the 'kube-system' namespace attributes restConfig, err := clientcmd.BuildConfigFromFlags("", s.cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { - panic(err) + return fmt.Errorf("Failed to build kubeconfig admin path: %v", err) } coreClient := clientv1.NewForConfigOrDie(rest.AddUserAgent(restConfig, "core-agent")) namespace, err := coreClient.Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}) if err != nil { - panic(err) + return fmt.Errorf("Failed to read 'kube-system' namespace attributes: %v", err) } // Use the 'kube-system' namespace metadata UID as the MicroShift Cluster ID clusterID := string(namespace.ObjectMeta.UID) // Write /cluster-id file if it does not already exist - initClusterIDFile(clusterID) + err = initClusterIDFile(clusterID) + if err != nil { + return fmt.Errorf("Failed to initialize cluster ID file: %v", err) + } // Log the cluster ID klog.Infof("MicroShift Cluster ID: %v", clusterID) return ctx.Err() } -func initClusterIDFile(clusterID string) { +func initClusterIDFile(clusterID string) error { // The location of the cluster ID file fileName := filepath.Join(config.DataDir, "cluster-id") // Do not create the cluster ID file if it already exists _, err := os.Stat(fileName) if !os.IsNotExist(err) { - return + return nil } // Write the cluster ID to a new file klog.Infof("Writing MicroShift Cluster ID '%v' to '%v'", clusterID, fileName) - err = os.WriteFile(fileName, []byte(clusterID), 0400) - if err != nil { - panic(err) - } + return os.WriteFile(fileName, []byte(clusterID), 0400) } From 10d4db73e34d70933611537578ac9425c8678164 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Tue, 30 Jan 2024 06:37:49 +0000 Subject: [PATCH 5/8] Remove unnecessary test from the cluster ID suite --- test/suites/osconfig/clusterid.robot | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/suites/osconfig/clusterid.robot b/test/suites/osconfig/clusterid.robot index 9048e95e95c..026b2d4f219 100644 --- a/test/suites/osconfig/clusterid.robot +++ b/test/suites/osconfig/clusterid.robot @@ -18,13 +18,6 @@ ${CLUSTERID_NS} kube-system *** Test Cases *** -Compare Cluster ID From Namespace And File - [Documentation] Verify that cluster ID is the same when read - ... from namespace and file - ${name_id}= Get MicroShift Cluster ID From Namespace - ${file_id}= Get MicroShift Cluster ID From File - Should Be Equal As Strings ${name_id} ${file_id} - Verify Cluster ID Change For New Database [Documentation] Verify that cluster ID changes after MicroShift ... database is cleaned and service restarted From 303e4368e05163b51ee9d4aff510323c35da3383 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 31 Jan 2024 07:26:21 +0000 Subject: [PATCH 6/8] Rewrite cluster ID if manually tampered by a user --- pkg/controllers/clusterid.go | 23 ++++++++++++++++++----- test/suites/osconfig/clusterid.robot | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/clusterid.go b/pkg/controllers/clusterid.go index c8d1b789c95..dfd8eca9bc4 100644 --- a/pkg/controllers/clusterid.go +++ b/pkg/controllers/clusterid.go @@ -17,6 +17,7 @@ package controllers import ( "context" + "fmt" "os" "path/filepath" @@ -51,20 +52,21 @@ func (s *ClusterID) Run(ctx context.Context, ready chan<- struct{}, stopped chan // Read the 'kube-system' namespace attributes restConfig, err := clientcmd.BuildConfigFromFlags("", s.cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { - return fmt.Errorf("Failed to build kubeconfig admin path: %v", err) + return fmt.Errorf("failed to build kubeconfig admin path: %v", err) } coreClient := clientv1.NewForConfigOrDie(rest.AddUserAgent(restConfig, "core-agent")) namespace, err := coreClient.Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}) if err != nil { - return fmt.Errorf("Failed to read 'kube-system' namespace attributes: %v", err) + return fmt.Errorf("failed to read 'kube-system' namespace attributes: %v", err) } // Use the 'kube-system' namespace metadata UID as the MicroShift Cluster ID clusterID := string(namespace.ObjectMeta.UID) // Write /cluster-id file if it does not already exist + // or has inconsistent contents err = initClusterIDFile(clusterID) if err != nil { - return fmt.Errorf("Failed to initialize cluster ID file: %v", err) + return fmt.Errorf("failed to initialize cluster ID file: %v", err) } // Log the cluster ID klog.Infof("MicroShift Cluster ID: %v", clusterID) @@ -76,10 +78,21 @@ func initClusterIDFile(clusterID string) error { // The location of the cluster ID file fileName := filepath.Join(config.DataDir, "cluster-id") - // Do not create the cluster ID file if it already exists + // Read and verify the cluster ID file if it already exists, + // logging a warning if the cluster ID is inconsistent _, err := os.Stat(fileName) if !os.IsNotExist(err) { - return nil + data, err := os.ReadFile(fileName) + if err != nil { + // Ignore the error, the file will be overwritten + klog.Warningf("Failed to read '%v' file: %v", fileName, err) + } else { + // Return if the cluster ID is consistent + if string(data) == clusterID { + return nil + } + klog.Warningf("Overwriting an inconsistent MicroShift Cluster ID '%v' in '%v' file", string(data), fileName) + } } // Write the cluster ID to a new file diff --git a/test/suites/osconfig/clusterid.robot b/test/suites/osconfig/clusterid.robot index 026b2d4f219..c48d20000e1 100644 --- a/test/suites/osconfig/clusterid.robot +++ b/test/suites/osconfig/clusterid.robot @@ -34,6 +34,17 @@ Verify Cluster ID Change For New Database Should Not Be Equal As Strings ${old_nid} ${new_nid} Should Not Be Equal As Strings ${old_fid} ${new_fid} +Verify Inconsistent Cluster ID Recovery + [Documentation] Verify that cluster ID is correctly rewritten on the + ... service restart after manual tampering by a user. + + Tamper With Cluster ID File + Restart MicroShift + + ${nid}= Get MicroShift Cluster ID From Namespace + ${fid}= Get MicroShift Cluster ID From File + Should Be Equal As Strings ${nid} ${fid} + *** Keywords *** Setup @@ -71,3 +82,11 @@ Get MicroShift Cluster ID From Namespace Should Not Be Empty ${clusterid} RETURN ${clusterid} + +Tamper With Cluster ID File + [Documentation] Append invalid characters to the cluster ID file. + ${stdout} ${stderr} ${rc}= Execute Command + ... echo -n 123 >> ${CLUSTERID_FILE} + ... sudo=True return_rc=True return_stdout=True return_stderr=True + + Should Be Equal As Integers 0 ${rc} From 12c4d4576635f3bab00803d90234dd8344261606 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 3 Feb 2024 09:47:55 +0000 Subject: [PATCH 7/8] Optimize the cluster ID creation logic --- pkg/controllers/clusterid.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/controllers/clusterid.go b/pkg/controllers/clusterid.go index dfd8eca9bc4..334233bdbe7 100644 --- a/pkg/controllers/clusterid.go +++ b/pkg/controllers/clusterid.go @@ -80,22 +80,20 @@ func initClusterIDFile(clusterID string) error { // Read and verify the cluster ID file if it already exists, // logging a warning if the cluster ID is inconsistent - _, err := os.Stat(fileName) - if !os.IsNotExist(err) { - data, err := os.ReadFile(fileName) - if err != nil { - // Ignore the error, the file will be overwritten - klog.Warningf("Failed to read '%v' file: %v", fileName, err) - } else { - // Return if the cluster ID is consistent - if string(data) == clusterID { - return nil - } - klog.Warningf("Overwriting an inconsistent MicroShift Cluster ID '%v' in '%v' file", string(data), fileName) + data, err := os.ReadFile(fileName) + if err != nil && !os.IsNotExist(err) { + // File exists, but cannot be read + return err + } + if len(data) > 0 { + if string(data) == clusterID { + // Consistent cluster ID file exists + return nil } + klog.Warningf("Overwriting an inconsistent MicroShift Cluster ID '%v' in '%v' file", string(data), fileName) } - // Write the cluster ID to a new file + // Write a new cluster ID file klog.Infof("Writing MicroShift Cluster ID '%v' to '%v'", clusterID, fileName) return os.WriteFile(fileName, []byte(clusterID), 0400) } From f9e7df26f3a9bb176f40c15e51e338b096817059 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Mon, 5 Feb 2024 16:31:23 +0200 Subject: [PATCH 8/8] Fix cluster id file tampering in the test --- test/suites/osconfig/clusterid.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suites/osconfig/clusterid.robot b/test/suites/osconfig/clusterid.robot index c48d20000e1..2eaf3f23d78 100644 --- a/test/suites/osconfig/clusterid.robot +++ b/test/suites/osconfig/clusterid.robot @@ -86,7 +86,7 @@ Get MicroShift Cluster ID From Namespace Tamper With Cluster ID File [Documentation] Append invalid characters to the cluster ID file. ${stdout} ${stderr} ${rc}= Execute Command - ... echo -n 123 >> ${CLUSTERID_FILE} + ... sed -i '$ s/$/123/' ${CLUSTERID_FILE} ... sudo=True return_rc=True return_stdout=True return_stderr=True Should Be Equal As Integers 0 ${rc}