From 8edf53231a50a9354090473ff44a230dd58a918e Mon Sep 17 00:00:00 2001 From: Rogger Vasquez Date: Tue, 7 Jun 2022 18:26:24 -0500 Subject: [PATCH] rpk: remove viper and mapstructure This commit removes viper and mapstructure These are all the required changes to handle the parsing, weak typing and writes to config file. From now on we will print warning to users so they know what needs to be changed in the config file before we deprecate this weak typing. This also include Yaml V3 upgrade, one notable change is the indent in yaml files. --- conf/redpanda.yaml | 8 +- src/go/k8s/cmd/configurator/main.go | 4 +- src/go/k8s/go.mod | 11 +- src/go/k8s/go.sum | 19 +- .../configuration/configuration_modes.go | 20 +- src/go/rpk/go.mod | 13 +- src/go/rpk/go.sum | 23 +- .../rpk/pkg/cli/cmd/cluster/config/reset.go | 2 +- src/go/rpk/pkg/cli/cmd/config.go | 5 +- src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go | 2 +- src/go/rpk/pkg/cli/cmd/generate/prometheus.go | 2 +- src/go/rpk/pkg/cli/cmd/mode.go | 6 +- src/go/rpk/pkg/cli/cmd/redpanda.go | 13 +- src/go/rpk/pkg/cli/cmd/redpanda/config.go | 174 +- .../rpk/pkg/cli/cmd/redpanda/config_test.go | 237 +-- src/go/rpk/pkg/cli/cmd/redpanda/mode.go | 26 +- src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go | 76 +- src/go/rpk/pkg/cli/cmd/redpanda/start.go | 95 +- src/go/rpk/pkg/cli/cmd/redpanda/start_test.go | 198 +- src/go/rpk/pkg/cli/cmd/redpanda/stop.go | 22 +- src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go | 5 +- src/go/rpk/pkg/cli/cmd/root.go | 3 +- src/go/rpk/pkg/cli/cmd/root_darwin.go | 5 +- src/go/rpk/pkg/cli/cmd/root_linux.go | 15 +- src/go/rpk/pkg/cli/cmd/start.go | 7 +- src/go/rpk/pkg/cli/cmd/stop.go | 5 +- src/go/rpk/pkg/config/config.go | 325 +--- src/go/rpk/pkg/config/config_test.go | 1618 +++-------------- src/go/rpk/pkg/config/find.go | 70 - src/go/rpk/pkg/config/find_test.go | 83 - src/go/rpk/pkg/config/license.go | 87 - src/go/rpk/pkg/config/license_test.go | 117 -- src/go/rpk/pkg/config/manager.go | 425 ----- src/go/rpk/pkg/config/params.go | 168 +- src/go/rpk/pkg/config/params_test.go | 99 +- src/go/rpk/pkg/config/schema.go | 188 +- src/go/rpk/pkg/config/weak.go | 63 +- src/go/rpk/pkg/config/weak_test.go | 159 +- src/go/rpk/pkg/plugin/manifest.go | 2 +- src/go/rpk/pkg/tuners/iotune/data.go | 2 +- src/go/rpk/pkg/tuners/redpanda_checkers.go | 2 +- src/go/rpk/pkg/yaml/yaml.go | 36 - tests/rptest/tests/rpk_config_test.py | 118 +- 43 files changed, 1285 insertions(+), 3273 deletions(-) delete mode 100644 src/go/rpk/pkg/config/find.go delete mode 100644 src/go/rpk/pkg/config/find_test.go delete mode 100644 src/go/rpk/pkg/config/license.go delete mode 100644 src/go/rpk/pkg/config/license_test.go delete mode 100644 src/go/rpk/pkg/config/manager.go delete mode 100644 src/go/rpk/pkg/yaml/yaml.go diff --git a/conf/redpanda.yaml b/conf/redpanda.yaml index ae98dc3d4c375..fee180320f1ae 100644 --- a/conf/redpanda.yaml +++ b/conf/redpanda.yaml @@ -52,15 +52,15 @@ schema_registry: rpk: # TLS configuration. - tls: + #tls: # The path to the root CA certificate (PEM) - truststore_file: "" + #truststore_file: "" # The path to the client certificate (PEM) - cert_file: "" + #cert_file: "" # The path to the client certificate key (PEM) - key_file: "" + #key_file: "" # Enables sending environment and resource usage data to Vectorized. enable_usage_stats: true diff --git a/src/go/k8s/cmd/configurator/main.go b/src/go/k8s/cmd/configurator/main.go index 8175c337f0c91..34cf8db968865 100644 --- a/src/go/k8s/cmd/configurator/main.go +++ b/src/go/k8s/cmd/configurator/main.go @@ -99,8 +99,8 @@ func main() { log.Print(c.String()) fs := afero.NewOsFs() - mgr := config.NewManager(fs) - cfg, err := mgr.Read(path.Join(c.configSourceDir, "redpanda.yaml")) + p := config.Params{ConfigPath: path.Join(c.configSourceDir, "redpanda.yaml")} + cfg, err := p.Load(fs) if err != nil { log.Fatalf("%s", fmt.Errorf("unable to read the redpanda configuration file: %w", err)) } diff --git a/src/go/k8s/go.mod b/src/go/k8s/go.mod index 82b88bb2169a8..9d4f2a1b28662 100644 --- a/src/go/k8s/go.mod +++ b/src/go/k8s/go.mod @@ -15,7 +15,7 @@ require ( github.com/redpanda-data/redpanda/src/go/rpk v0.0.0-00010101000000-000000000000 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.21.4 k8s.io/apimachinery v0.21.4 k8s.io/client-go v0.21.4 @@ -48,16 +48,12 @@ require ( github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -65,12 +61,8 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/sethgrid/pester v1.1.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect - github.com/spf13/cast v1.3.0 // indirect github.com/spf13/cobra v1.1.3 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/twmb/tlscfg v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -86,7 +78,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.52.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.21.4 // indirect diff --git a/src/go/k8s/go.sum b/src/go/k8s/go.sum index 9d10638d2d1f9..13b15e01adc38 100644 --- a/src/go/k8s/go.sum +++ b/src/go/k8s/go.sum @@ -51,7 +51,6 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -347,7 +346,6 @@ github.com/googleapis/gnostic v0.5.4/go.mod h1:TRWw1s4gxBGjSe301Dai3c7wXJAZy57+/ github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -393,7 +391,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -431,7 +428,6 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -464,7 +460,6 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -503,8 +498,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -579,7 +572,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pavel-v-chernykh/keystore-go/v4 v4.1.0/go.mod h1:2ejgys4qY+iNVW1IittZhyRYA6MNv8TgM6VHqojbB9g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -654,9 +646,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= @@ -665,7 +655,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -673,7 +662,6 @@ github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -682,7 +670,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -699,7 +686,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -1123,8 +1109,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4= -gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1145,8 +1129,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/k8s/pkg/resources/configuration/configuration_modes.go b/src/go/k8s/pkg/resources/configuration/configuration_modes.go index 30d46e2898a55..8833aa53ccfce 100644 --- a/src/go/k8s/pkg/resources/configuration/configuration_modes.go +++ b/src/go/k8s/pkg/resources/configuration/configuration_modes.go @@ -14,9 +14,6 @@ import ( "reflect" "strconv" "strings" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/spf13/afero" ) // GlobalConfigurationMode changes the behavior of the global configuration when reading/writing properties. @@ -65,33 +62,20 @@ func (r globalConfigurationModeClassic) GetAdditionalRedpandaProperty( func (r globalConfigurationModeClassic) SetAdditionalFlatProperties( targetConfig *GlobalConfiguration, props map[string]string, ) error { - // all properties are node properties in the classic setting - mgr := config.NewManager(afero.NewOsFs()) - err := mgr.Merge(&targetConfig.NodeConfiguration) - if err != nil { - return fmt.Errorf("merging node configuration: %w", err) - } - // Add arbitrary parameters to configuration for k, v := range props { if builtInType(v) { - err = mgr.Set(k, v, "single") + err := targetConfig.NodeConfiguration.Set(k, v, "") if err != nil { return fmt.Errorf("setting built-in type: %w", err) } } else { - err = mgr.Set(k, v, "") + err := targetConfig.NodeConfiguration.Set(k, v, "") if err != nil { return fmt.Errorf("setting complex type: %w", err) } } } - - newRpCfg, err := mgr.Get() - if err != nil { - return fmt.Errorf("getting redpanda node configuration: %w", err) - } - targetConfig.NodeConfiguration = *newRpCfg return nil } diff --git a/src/go/rpk/go.mod b/src/go/rpk/go.mod index 454c82a4709a7..9fcaaf55d4daf 100644 --- a/src/go/rpk/go.mod +++ b/src/go/rpk/go.mod @@ -17,7 +17,6 @@ require ( github.com/google/uuid v1.1.1 github.com/hashicorp/go-multierror v1.1.0 github.com/lorenzosaino/go-sysctl v0.1.0 - github.com/mitchellh/mapstructure v1.4.1 github.com/olekukonko/tablewriter v0.0.1 github.com/opencontainers/image-spec v1.0.1 github.com/pkg/errors v0.9.1 @@ -29,7 +28,6 @@ require ( github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 github.com/tklauser/go-sysconf v0.1.0 github.com/twmb/franz-go v1.6.0 @@ -40,8 +38,7 @@ require ( golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -49,20 +46,17 @@ require ( github.com/containerd/containerd v1.4.8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/godbus/dbus/v5 v5.0.3 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/protobuf v1.4.2 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect @@ -71,12 +65,8 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tklauser/numcpus v0.1.0 // indirect golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect @@ -86,6 +76,5 @@ require ( google.golang.org/grpc v1.27.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.51.0 // indirect gotest.tools/v3 v3.0.3 // indirect ) diff --git a/src/go/rpk/go.sum b/src/go/rpk/go.sum index f96d58a2c356a..b83b8a7b83d4a 100644 --- a/src/go/rpk/go.sum +++ b/src/go/rpk/go.sum @@ -16,7 +16,6 @@ github.com/AlecAivazis/survey/v2 v2.3.2 h1:TqTB+aDDCLYhf9/bD2TwSO8u8jDSmMUd2SUVO github.com/AlecAivazis/survey/v2 v2.3.2/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= @@ -77,8 +76,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -123,7 +120,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -149,7 +145,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -164,7 +159,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -190,7 +184,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lorenzosaino/go-sysctl v0.1.0 h1:BfWlLYErjQeCb0TB3kzIq5nsVVjKqyT0NKvWqifz7gE= github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -216,8 +209,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -233,7 +224,6 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -275,9 +265,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= @@ -285,16 +273,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -304,7 +289,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tklauser/go-sysconf v0.1.0 h1:f4fOH/B3HZYy2wz7o9Vr5i4dBhOKcedooHq7/4c0hxQ= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -401,7 +385,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -490,18 +473,16 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go index e2bc13650ceb9..53d5b2e833196 100644 --- a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go +++ b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go @@ -16,7 +16,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func newForceResetCommand(fs afero.Fs) *cobra.Command { diff --git a/src/go/rpk/pkg/cli/cmd/config.go b/src/go/rpk/pkg/cli/cmd/config.go index 6061658cce7e1..1051630ed50b2 100644 --- a/src/go/rpk/pkg/cli/cmd/config.go +++ b/src/go/rpk/pkg/cli/cmd/config.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewConfigCommand(fs, mgr), + redpanda.NewConfigCommand(fs), "rpk redpanda config", ) } diff --git a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go index 0176dd8382a9d..e75dc1343899f 100644 --- a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go +++ b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go @@ -43,7 +43,7 @@ import ( "github.com/spf13/afero" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func executeBundle( diff --git a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go index 5d9a32f2c48e0..8d78a55234039 100644 --- a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go +++ b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/cobra" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type ScrapeConfig struct { diff --git a/src/go/rpk/pkg/cli/cmd/mode.go b/src/go/rpk/pkg/cli/cmd/mode.go index a300c74111a24..802d8b1b1b242 100644 --- a/src/go/rpk/pkg/cli/cmd/mode.go +++ b/src/go/rpk/pkg/cli/cmd/mode.go @@ -15,13 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewModeCommand(mgr), + redpanda.NewModeCommand(fs), "rpk redpanda mode", ) } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda.go b/src/go/rpk/pkg/cli/cmd/redpanda.go index 14d68a75320e4..e0c980a5cd3af 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda.go @@ -15,26 +15,23 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda/admin" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewRedpandaCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewRedpandaCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { command := &cobra.Command{ Use: "redpanda", Short: "Interact with a local Redpanda process", } - command.AddCommand(redpanda.NewStartCommand(fs, mgr, launcher)) - command.AddCommand(redpanda.NewStopCommand(fs, mgr)) + command.AddCommand(redpanda.NewStartCommand(fs, launcher)) + command.AddCommand(redpanda.NewStopCommand(fs)) command.AddCommand(redpanda.NewCheckCommand(fs)) command.AddCommand(redpanda.NewTuneCommand(fs)) - command.AddCommand(redpanda.NewModeCommand(mgr)) - command.AddCommand(redpanda.NewConfigCommand(fs, mgr)) + command.AddCommand(redpanda.NewModeCommand(fs)) + command.AddCommand(redpanda.NewConfigCommand(fs)) command.AddCommand(admin.NewCommand(fs)) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config.go b/src/go/rpk/pkg/cli/cmd/redpanda/config.go index b79a8d0f857a6..8412479faae90 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config.go @@ -17,74 +17,78 @@ import ( "fmt" "net" + "github.com/google/uuid" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" vnet "github.com/redpanda-data/redpanda/src/go/rpk/pkg/net" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" ) -const configFileFlag = "config" +const ( + configFileFlag = "config" + configFileFlagDesc = "Redpanda config file, if not set the file will be searched for in the default location" +) -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { root := &cobra.Command{ Use: "config ", Short: "Edit configuration.", } - root.AddCommand(set(fs, mgr)) - root.AddCommand(bootstrap(mgr)) - root.AddCommand(initNode(mgr)) + root.AddCommand(set(fs)) + root.AddCommand(bootstrap(fs)) + root.AddCommand(initNode(fs)) return root } -func set(fs afero.Fs, mgr config.Manager) *cobra.Command { +func set(fs afero.Fs) *cobra.Command { var ( format string configPath string ) c := &cobra.Command{ Use: "set ", - Short: "Set configuration values, such as the node IDs or the list of seed servers.", - Args: cobra.ExactArgs(2), - RunE: func(_ *cobra.Command, args []string) error { - var err error - key := args[0] - value := args[1] - if configPath == "" { - configPath, err = config.FindConfigFile(fs) - if err != nil { - return err - } - } - _, err = mgr.Read(configPath) - if err != nil { - return err - } - err = mgr.Set(key, value, format) - if err != nil { - return err + Short: "Set configuration values, such as the node IDs or the list of seed servers", + Long: `Set configuration values, such as the node IDs or the list of seed servers + +You can pass a key that represents the configuration property name, if it's a +nested property you should pass the group/category where it belongs, e.g: + + rpk redpanda config set redpanda.developer_mode true + +if --format is not used, rpk will use yaml as default, you can also pass +partial json/yaml config objects: + + rpk redpanda config set redpanda.rpc_server '{"address":"0.0.0.0","port":33145}' --format json +`, + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + + if format == "single" { + fmt.Println("'--format single' is deprecated, either remove it or use yaml/json") } - return mgr.WriteLoaded() + err = cfg.Set(args[0], args[1], format) + out.MaybeDie(err, "unable to set %q:%v", args[0], err) + + err = cfg.Write(fs) + out.MaybeDieErr(err) }, } - c.Flags().StringVar(&format, - "format", - "single", - "The value format. Can be 'single', for single values such as"+ - " '/etc/redpanda' or 100; and 'json' and 'yaml' when"+ - " partially or completely setting config objects", - ) + c.Flags().StringVar(&format, "format", "yaml", "Format of the value (yaml/json)") c.Flags().StringVar( &configPath, configFileFlag, "", - "Redpanda config file, if not set the file will be searched"+ - " for in the default location.", + configFileFlagDesc, ) return c } -func bootstrap(mgr config.Manager) *cobra.Command { +func bootstrap(fs afero.Fs) *cobra.Command { var ( ips []string self string @@ -93,44 +97,36 @@ func bootstrap(mgr config.Manager) *cobra.Command { ) c := &cobra.Command{ Use: "bootstrap --id [--self ] [--ips ]", - Short: "Initialize the configuration to bootstrap a cluster.", + Short: "Initialize the configuration to bootstrap a cluster", Long: helpBootstrap, - Args: cobra.OnlyValidArgs, - RunE: func(c *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + seeds, err := parseSeedIPs(ips) - if err != nil { - return err - } - var ownIP net.IP - if self != "" { - ownIP = net.ParseIP(self) - if ownIP == nil { - return fmt.Errorf("%s is not a valid IP", self) - } - } else { - ownIP, err = getOwnIP() - if err != nil { - return err - } - } - conf.Redpanda.ID = id - conf.Redpanda.RPCServer.Address = ownIP.String() - conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ + out.MaybeDieErr(err) + + ownIP, err := parseSelfIP(self) + out.MaybeDieErr(err) + + cfg.Redpanda.ID = id + cfg.Redpanda.RPCServer.Address = ownIP.String() + cfg.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultKafkaPort, }} - conf.Redpanda.AdminAPI = []config.NamedSocketAddress{{ + cfg.Redpanda.AdminAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultAdminPort, }} - conf.Redpanda.SeedServers = []config.SeedServer{} - conf.Redpanda.SeedServers = seeds - return mgr.Write(conf) + cfg.Redpanda.SeedServers = []config.SeedServer{} + cfg.Redpanda.SeedServers = seeds + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringSliceVar( @@ -143,8 +139,7 @@ func bootstrap(mgr config.Manager) *cobra.Command { &configPath, configFileFlag, "", - "Redpanda config file, if not set the file will be searched"+ - " for in the default location.", + configFileFlagDesc, ) c.Flags().StringVar( &self, @@ -162,34 +157,53 @@ func bootstrap(mgr config.Manager) *cobra.Command { return c } -func initNode(mgr config.Manager) *cobra.Command { +func initNode(fs afero.Fs) *cobra.Command { var configPath string c := &cobra.Command{ Use: "init", - Short: "Init the node after install, by setting the node's UUID.", - Args: cobra.OnlyValidArgs, - RunE: func(_ *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Short: "Init the node after install, by setting the node's UUID", + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + // Don't reset the node's UUID if it has already been set. - if conf.NodeUUID == "" { - return mgr.WriteNodeUUID(conf) + if cfg.NodeUUID == "" { + id, err := uuid.NewUUID() + out.MaybeDie(err, "error creating nodeUUID: %v", err) + cfg.NodeUUID = id.String() } - return nil + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringVar( &configPath, configFileFlag, "", - "Redpanda config file, if not set the file will be searched"+ - " for in the default location.", + configFileFlagDesc, ) return c } +func parseSelfIP(self string) (net.IP, error) { + if self != "" { + ownIP := net.ParseIP(self) + if ownIP == nil { + return nil, fmt.Errorf("%s is not a valid IP", self) + } + return ownIP, nil + } else { + ownIP, err := getOwnIP() + if err != nil { + return nil, err + } + return ownIP, nil + } +} + func parseSeedIPs(ips []string) ([]config.SeedServer, error) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port var seeds []config.SeedServer diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go index b69c0e536ac5b..7891c9b3dfed4 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go @@ -13,166 +13,15 @@ package redpanda import ( - "path/filepath" "strings" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) -func TestSetCmd(t *testing.T) { - tests := []struct { - name string - key string - value string - args []string - expected interface{} - expectErr bool - }{ - { - name: "it should set single integer fields", - key: "redpanda.node_id", - value: "54312", - expected: 54312, - }, - { - name: "it should set single float fields", - key: "redpanda.float_field", - value: "42.3", - expected: 42.3, - }, - { - name: "it should set single string fields", - key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", - expected: "'/var/lib/differentdir'", - }, - { - name: "it should set single bool fields", - key: "rpk.enable_usage_stats", - value: "true", - expected: true, - }, - { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, - args: []string{"--format", "yaml"}, - expected: map[string]interface{}{ - "enable_usage_stats": false, - "overprovisioned": false, - "tune_network": false, - "tune_disk_scheduler": false, - "tune_disk_write_cache": false, - "tune_disk_nomerges": false, - "tune_disk_irq": true, - "tune_cpu": false, - "tune_aio_events": false, - "tune_clocksource": false, - "tune_swappiness": false, - "tune_transparent_hugepages": false, - "enable_memory_locking": false, - "tune_fstrim": false, - "tune_coredump": false, - "tune_ballast_file": false, - "coredump_dir": "/var/lib/redpanda/coredump", - }, - }, - { - name: "it should partially set map fields (json)", - key: "redpanda.kafka_api", - value: `[{ - "address": "192.168.54.2", - "port": 9092 -}]`, - args: []string{"--format", "json"}, - expected: []interface{}{ - map[interface{}]interface{}{ - "port": 9092, - "address": "192.168.54.2", - }, - }, - }, - { - name: "it should fail if the new value is invalid", - key: "redpanda", - value: `{"data_directory": ""}`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (json)", - key: "redpanda", - value: `{"seed_servers": []`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (yaml)", - key: "redpanda", - value: `seed_servers: -- host: - address: "123.`, - args: []string{"--format", "yaml"}, - expectErr: true, - }, - { - name: "it should fail if the format isn't supported", - key: "redpanda", - value: `node_id=1`, - args: []string{"--format", "toml"}, - expectErr: true, - }, - { - name: "it should fail if no key is passed", - value: `node_id=1`, - expectErr: true, - }, - { - name: "it should fail if no value is passed", - key: "rpk.tune_coredump", - expectErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - - c := NewConfigCommand(fs, mgr) - args := []string{"set"} - if tt.key != "" { - args = append(args, tt.key) - } - if tt.value != "" { - args = append(args, tt.value) - } - c.SetArgs(append(args, tt.args...)) - err = c.Execute() - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) - val := v.Get(tt.key) - require.Exactly(t, tt.expected, val) - }) - } -} - func TestBootstrap(t *testing.T) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port tests := []struct { @@ -252,39 +101,13 @@ func TestBootstrap(t *testing.T) { self: "192.168.34.5", id: "1", }, - { - name: "it should fail if any of the --ips IPs isn't valid", - ips: []string{"187.89.9", "192.168.34.5", "192.168.45.8"}, - self: "192.168.34.5", - id: "1", - expectedErr: `invalid host "187.89.9" does not match "host", nor "host:port", nor "scheme://host:port"`, - }, - { - name: "it should fail if --self isn't a valid IP", - ips: []string{"187.89.9.78", "192.168.34.5", "192.168.45.8"}, - self: "www.host.com", - id: "1", - expectedErr: "www.host.com is not a valid IP", - }, - { - name: "it should fail if --id isn't passed", - expectedErr: "required flag(s) \"id\" not set", - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - configPath, err := filepath.Abs("./redpanda.yaml") - require.NoError(t, err) fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - err = fs.MkdirAll( - filepath.Dir(configPath), - 0o644, - ) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"bootstrap", "--config", configPath} + c := bootstrap(fs) + var args []string if len(tt.ips) != 0 { args = append( args, @@ -299,15 +122,15 @@ func TestBootstrap(t *testing.T) { args = append(args, "--id", tt.id) } c.SetArgs(args) - err = c.Execute() + err := c.Execute() if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return } require.NoError(t, err) - _, err = fs.Stat(configPath) + _, err = fs.Stat(config.Default().ConfigFile) require.NoError(t, err) - conf, err := mgr.Read(configPath) + conf, err := new(config.Params).Load(fs) require.NoError(t, err) require.Equal(t, tt.self, conf.Redpanda.RPCServer.Address) @@ -327,25 +150,37 @@ func TestBootstrap(t *testing.T) { } func TestInitNode(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"init"} - c.SetArgs(args) + for _, test := range []struct { + name string + prevID string + }{ + {name: "without UUID"}, + {name: "with UUID", prevID: "my_id"}, + } { + t.Run(test.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + c := config.Default() + if test.prevID != "" { + c.NodeUUID = test.prevID + } - err = c.Execute() - require.NoError(t, err) + bs, err := yaml.Marshal(c) + require.NoError(t, err) + err = afero.WriteFile(fs, c.ConfigFile, bs, 0o644) + require.NoError(t, err) + + cmd := initNode(fs) + err = cmd.Execute() + require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) + conf, err := new(config.Params).Load(fs) + require.NoError(t, err) - val := v.Get("node_uuid") - require.NotEmpty(t, val) + if test.prevID != "" { + require.Exactly(t, conf.NodeUUID, test.prevID) + } else { + require.NotEmpty(t, conf.NodeUUID) + } + }) + } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go index 0378d76fd86f1..8019b1fcc6095 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go @@ -17,11 +17,12 @@ import ( "strings" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - log "github.com/sirupsen/logrus" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { var configFile string command := &cobra.Command{ Use: "mode ", @@ -33,9 +34,10 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { } return nil }, - RunE: func(_ *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { // Safe to access args[0] because it was validated in Args - return executeMode(mgr, configFile, args[0]) + err := executeMode(fs, cmd, args[0]) + out.MaybeDieErr(err) }, } command.Flags().StringVar( @@ -48,15 +50,21 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { return command } -func executeMode(mgr config.Manager, configFile string, mode string) error { - conf, err := mgr.FindOrGenerate(configFile) +func executeMode(fs afero.Fs, cmd *cobra.Command, mode string) error { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + if err != nil { + return fmt.Errorf("unable to load config: %v", err) + } + cfg, err = config.SetMode(mode, cfg) if err != nil { return err } - conf, err = config.SetMode(mode, conf) + + fmt.Printf("Writing %q mode defaults to %q\n", mode, cfg.ConfigFile) + err = cfg.Write(fs) if err != nil { return err } - log.Infof("Writing '%s' mode defaults to '%s'", mode, conf.ConfigFile) - return mgr.Write(conf) + return nil } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go index b65f4be4d1b21..5d556bffc1329 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go @@ -13,17 +13,14 @@ package redpanda import ( - "bytes" - "fmt" "os" - "strings" + "path/filepath" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func fillRpkConfig(path, mode string) *config.Config { @@ -45,23 +42,21 @@ func fillRpkConfig(path, mode string) *config.Config { Overprovisioned: !val, TuneBallastFile: val, } + // Unset defaults that get added after command execution, needed to compare + // expected config with loaded config. + conf.Rpk.KafkaAPI = config.RpkKafkaAPI{Brokers: []string{"0.0.0.0:9092"}} + conf.Rpk.AdminAPI = config.RpkAdminAPI{Addresses: []string{"127.0.0.1:9644"}} return conf } func TestModeCommand(t *testing.T) { configPath := "/etc/redpanda/redpanda.yaml" - dir, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - wdConfigPath := fmt.Sprintf("%s/redpanda.yaml", dir) tests := []struct { - name string - args []string - before func(afero.Fs) (string, error) - expectedConfig *config.Config - expectedOutput string - expectedErrMsg string + name string + args []string + before func(afero.Fs) (string, error) + exp *config.Config + expErr bool }{ { name: "development mode should disable all fields in the rpk config", @@ -73,9 +68,7 @@ func TestModeCommand(t *testing.T) { } return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, - expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'development' mode defaults to '%s'", configPath), - expectedErrMsg: "", + exp: fillRpkConfig(configPath, config.ModeDev), }, { name: "production mode should enable all fields in the rpk config", @@ -87,9 +80,7 @@ func TestModeCommand(t *testing.T) { } return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, - expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'production' mode defaults to '%s'", configPath), - expectedErrMsg: "", + exp: fillRpkConfig(configPath, config.ModeProd), }, { name: "the development mode alias, 'dev', should work the same", @@ -101,9 +92,7 @@ func TestModeCommand(t *testing.T) { } return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, - expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'dev' mode defaults to '%s'", configPath), - expectedErrMsg: "", + exp: fillRpkConfig(configPath, config.ModeDev), }, { name: "the production mode alias, 'prod', should work the same", @@ -115,9 +104,7 @@ func TestModeCommand(t *testing.T) { } return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, - expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", + exp: fillRpkConfig(configPath, config.ModeProd), }, { name: "mode should work if --config isn't passed, but the file is in /etc/redpanda/redpanda.yaml", @@ -129,47 +116,44 @@ func TestModeCommand(t *testing.T) { } return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, - expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", + exp: fillRpkConfig(configPath, config.ModeProd), }, { name: "mode lists the available modes if the one passed is not valid", args: []string{"invalidmode"}, before: func(fs afero.Fs) (string, error) { + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + wdConfigPath := filepath.Join(dir, "/redpanda.yaml") bs, err := yaml.Marshal(fillRpkConfig(wdConfigPath, config.ModeDev)) if err != nil { return "", err } return wdConfigPath, afero.WriteFile(fs, wdConfigPath, bs, 0o644) }, - expectedOutput: "", - expectedErrMsg: "'invalidmode' is not a supported mode. Available modes: dev, development, prod, production", + expErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - path, err := tt.before(fs) + _, err := tt.before(fs) require.NoError(t, err) - var out bytes.Buffer - cmd := NewModeCommand(mgr) + cmd := NewModeCommand(fs) cmd.SetArgs(tt.args) - logrus.SetOutput(&out) - err = cmd.Execute() - if tt.expectedErrMsg != "" { - require.EqualError(t, err, tt.expectedErrMsg) + err = executeMode(fs, cmd, tt.args[0]) + if tt.expErr && err != nil { return } require.NoError(t, err) - output := out.String() - require.Contains(t, strings.TrimSpace(output), tt.expectedOutput) - mgr = config.NewManager(fs) - conf, err := mgr.Read(path) + + conf, err := new(config.Params).Load(fs) require.NoError(t, err) - require.Exactly(t, tt.expectedConfig, conf) + + require.Exactly(t, tt.exp, conf.File()) }) } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start.go b/src/go/rpk/pkg/cli/cmd/redpanda/start.go index b362bfad50908..fe9f5e779dcd6 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start.go @@ -113,9 +113,7 @@ func parseConfigKvs(args []string) ([]string, []string) { return kvs, args } -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { prestartCfg := prestartConfig{} var ( configFile string @@ -143,25 +141,25 @@ func NewStartCommand( // (POSIX standard) UnknownFlags: true, }, - RunE: func(ccmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { // --set flags have to be parsed by hand because pflag (the // underlying flag-parsing lib used by cobra) uses a CSV parser // for list flags, and since JSON often contains commas, it // blows up when there's a JSON object. configKvs, filteredArgs := parseConfigKvs(os.Args) - conf, err := mgr.FindOrGenerate(configFile) + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) if err != nil { - return err + return fmt.Errorf("unable to load config file: %s", err) } if len(configKvs) > 0 { - conf, err = setConfig(mgr, configKvs) - if err != nil { + if err = setConfig(cfg, configKvs); err != nil { return err } } - updateConfigWithFlags(conf, ccmd.Flags()) + updateConfigWithFlags(cfg, cmd.Flags()) env := api.EnvironmentPayload{} if len(seeds) == 0 { @@ -177,11 +175,11 @@ func NewStartCommand( } seedServers, err := parseSeeds(seeds) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(seedServers) != 0 { - conf.Redpanda.SeedServers = seedServers + cfg.Redpanda.SeedServers = seedServers } kafkaAddr = stringSliceOr( @@ -196,11 +194,11 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(kafkaAPI) > 0 { - conf.Redpanda.KafkaAPI = kafkaAPI + cfg.Redpanda.KafkaAPI = kafkaAPI } proxyAddr = stringSliceOr( @@ -215,14 +213,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(proxyAPI) > 0 { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.PandaproxyAPI = proxyAPI + cfg.Pandaproxy.PandaproxyAPI = proxyAPI } schemaRegAddr = stringSliceOr( @@ -237,14 +235,14 @@ func NewStartCommand( config.DefaultSchemaRegPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(schemaRegAPI) > 0 { - if conf.SchemaRegistry == nil { - conf.SchemaRegistry = config.Default().SchemaRegistry + if cfg.SchemaRegistry == nil { + cfg.SchemaRegistry = config.Default().SchemaRegistry } - conf.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI + cfg.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI } rpcAddr = stringOr( @@ -256,11 +254,11 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if rpcServer != nil { - conf.Redpanda.RPCServer = *rpcServer + cfg.Redpanda.RPCServer = *rpcServer } advertisedKafka = stringSliceOr( @@ -275,11 +273,12 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - if advKafkaAPI != nil { - conf.Redpanda.AdvertisedKafkaAPI = advKafkaAPI + + if len(advKafkaAPI) > 0 { + cfg.Redpanda.AdvertisedKafkaAPI = advKafkaAPI } advertisedProxy = stringSliceOr( @@ -294,14 +293,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advProxyAPI != nil { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI + cfg.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI } advertisedRPC = stringOr( @@ -313,53 +312,53 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advRPCApi != nil { - conf.Redpanda.AdvertisedRPCAPI = advRPCApi + cfg.Redpanda.AdvertisedRPCAPI = advRPCApi } installDirectory, err := cli.GetOrFindInstallDir(fs, installDirFlag) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } rpArgs, err := buildRedpandaFlags( fs, - conf, + cfg, filteredArgs, sFlags, - ccmd.Flags(), + cmd.Flags(), !prestartCfg.checkEnabled, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } checkPayloads, tunerPayloads, err := prestart( fs, rpArgs, - conf, + cfg, prestartCfg, timeout, ) env.Checks = checkPayloads env.Tuners = tunerPayloads if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - err = mgr.Write(conf) + err = cfg.Write(fs) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, nil) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, nil) rpArgs.ExtraArgs = args - log.Info(common.FeedbackMsg) - log.Info("Starting redpanda...") + fmt.Println(common.FeedbackMsg) + fmt.Println("Starting redpanda...") return launcher.Start(installDirectory, rpArgs) }, } @@ -640,21 +639,21 @@ func mergeFlags( return current } -func setConfig(mgr config.Manager, configKvs []string) (*config.Config, error) { +func setConfig(cfg *config.Config, configKvs []string) error { for _, rawKv := range configKvs { parts := strings.SplitN(rawKv, "=", 2) if len(parts) < 2 { - return nil, fmt.Errorf( + return fmt.Errorf( "key-value pair '%s' is not formatted as expected (k=v)", rawKv, ) } - err := mgr.Set(parts[0], parts[1], "") + err := cfg.Set(parts[0], parts[1], "") if err != nil { - return nil, err + return err } } - return mgr.Get() + return nil } func resolveWellKnownIo( diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go index 5fb8afc148759..ac7a3b536760b 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go @@ -14,7 +14,9 @@ package redpanda import ( "bytes" + "net" "os" + "strconv" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" @@ -171,7 +173,7 @@ func TestStartCommand(t *testing.T) { 0o755, ) }, - expectedErrMsg: "An error happened while trying to read /etc/redpanda/redpanda.yaml: While parsing config: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml` into map[string]interface {}", + expectedErrMsg: "unable to load config file: unable to yaml decode /etc/redpanda/redpanda.yaml: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml`", }, { name: "should generate the config at the given path if it doesn't exist", args: []string{ @@ -191,11 +193,15 @@ func TestStartCommand(t *testing.T) { "The config should have been created at '%s'", path, ) - defaultConf := config.Default() - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + c := config.Default() + // Adding unset default that get added on first load. + b0 := c.Redpanda.KafkaAPI[0] + c.Rpk.KafkaAPI.Brokers = []string{net.JoinHostPort(b0.Address, strconv.Itoa(b0.Port))} + c.Rpk.AdminAPI.Addresses = []string{"127.0.0.1:9644"} + + conf, err := new(config.Params).Load(fs) require.NoError(st, err) - require.Exactly(st, defaultConf, conf) + require.Exactly(st, c, conf.File()) }, }, { name: "it should write the given config file path", @@ -208,8 +214,8 @@ func TestStartCommand(t *testing.T) { }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} // In command execution this will be done by with ParamsFromCommand + conf, err := p.Load(fs) require.NoError(st, err) require.Exactly(st, path, conf.ConfigFile) }, @@ -255,9 +261,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -321,9 +326,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -375,9 +379,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) // The value set through the --kafka-addr flag should // have been picked. @@ -396,9 +399,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -408,13 +409,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) - conf := config.Default() - return mgr.Write(conf) + cfg := config.Default() + return cfg.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -426,9 +425,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, 34, conf.Redpanda.ID) }, @@ -439,9 +436,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, config.Default().Redpanda.ID, conf.Redpanda.ID) @@ -452,14 +447,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.ID = 98 - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -475,9 +468,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, "aws:i3xlarge:default", conf.Rpk.WellKnownIo) }, @@ -488,14 +479,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.WellKnownIo = "gcp:n2standard:ssd" - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -513,9 +502,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, false, conf.Rpk.Overprovisioned) @@ -526,13 +513,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -549,9 +534,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, true, conf.Rpk.EnableMemoryLocking) @@ -563,14 +546,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.EnableMemoryLocking = true - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -586,8 +567,7 @@ func TestStartCommand(t *testing.T) { "--seeds", "192.168.34.32:33145,somehost:54321,justahostnoport", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ Host: config.SocketAddress{ @@ -620,8 +600,7 @@ func TestStartCommand(t *testing.T) { "-s", "192.168.123.32:33146,host", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ Host: config.SocketAddress{ @@ -659,8 +638,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_SEEDS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ Host: config.SocketAddress{ @@ -686,7 +664,6 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.SeedServers = []config.SeedServer{{ Host: config.SocketAddress{ @@ -694,11 +671,10 @@ func TestStartCommand(t *testing.T) { Port: 33146, }, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ Host: config.SocketAddress{ @@ -732,8 +708,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -753,8 +728,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -787,8 +761,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "host", @@ -807,17 +780,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.RPCServer = config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.33.33", @@ -837,8 +808,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -858,8 +828,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -879,8 +848,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -901,8 +869,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32,host:9092", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -939,8 +906,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -959,17 +925,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -989,8 +953,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1010,8 +973,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1044,8 +1006,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1064,17 +1025,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedKafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -1094,8 +1053,7 @@ func TestStartCommand(t *testing.T) { "--advertise-pandaproxy-addr", "192.168.34.32:8083", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1121,8 +1079,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_PANDAPROXY_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1142,8 +1099,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1163,8 +1119,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1197,8 +1152,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "host", @@ -1217,17 +1171,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedRPCAPI = &config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.33.33", @@ -1246,10 +1198,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--overprovisioned", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--overprovisioned"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--overprovisioned' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--overprovisioned' directly to `rpk start`.", }, { @@ -1258,10 +1209,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--smp", "1", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--smp=1"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--smp' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--smp' directly to `rpk start`.", }, { @@ -1270,10 +1220,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--memory", "2G", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--memory=1G"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--memory' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--memory' directly to `rpk start`.", }, { @@ -1282,12 +1231,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--smp=3", "--smp=55", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1302,12 +1250,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--logger-log-level=archival=debug:cloud_storage=debug", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1362,7 +1309,6 @@ func TestStartCommand(t *testing.T) { defer tt.after() } fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) var launcher redpanda.Launcher = &noopLauncher{} if tt.launcher != nil { launcher = tt.launcher @@ -1372,11 +1318,11 @@ func TestStartCommand(t *testing.T) { } var out bytes.Buffer logrus.SetOutput(&out) - c := NewStartCommand(fs, mgr, launcher) + c := NewStartCommand(fs, launcher) c.SetArgs(tt.args) err := c.Execute() if tt.expectedErrMsg != "" { - require.EqualError(st, err, tt.expectedErrMsg) + require.Contains(st, err.Error(), tt.expectedErrMsg) return } require.NoError(st, err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go index 5f452ba823732..837b3f6177e28 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go @@ -20,13 +20,14 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/os" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" log "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { var ( configFile string timeout time.Duration @@ -38,8 +39,13 @@ func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { first sends SIGINT, and waits for the specified timeout. Then, if redpanda hasn't stopped, it sends SIGTERM. Lastly, it sends SIGKILL if it's still running.`, - RunE: func(ccmd *cobra.Command, args []string) error { - return executeStop(fs, mgr, configFile, timeout) + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + + err = executeStop(fs, cfg, timeout) + out.MaybeDieErr(err) }, } command.Flags().StringVar( @@ -63,14 +69,8 @@ running.`, return command } -func executeStop( - fs afero.Fs, mgr config.Manager, configFile string, timeout time.Duration, -) error { - conf, err := mgr.ReadOrFind(configFile) - if err != nil { - return err - } - pidFile := conf.PIDFile() +func executeStop(fs afero.Fs, cfg *config.Config, timeout time.Duration) error { + pidFile := cfg.PIDFile() isLocked, err := os.CheckLocked(pidFile) if err != nil { log.Debugf("error checking if the PID file is locked: %v", err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go index f6e4319ba4f38..7b7cc1a48be74 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go @@ -55,7 +55,6 @@ func TestStopCommand(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) conf := config.Default() command := baseCommand // trap the signals we want to ignore, to check that the @@ -76,11 +75,11 @@ func TestStopCommand(t *testing.T) { conf.PIDFile(), ) require.NoError(t, err) - err = mgr.Write(conf) + err = conf.Write(fs) require.NoError(t, err) var out bytes.Buffer - c := cmd.NewStopCommand(fs, mgr) + c := cmd.NewStopCommand(fs) args := append([]string{"--config", conf.ConfigFile}, tt.args...) c.SetArgs(args) diff --git a/src/go/rpk/pkg/cli/cmd/root.go b/src/go/rpk/pkg/cli/cmd/root.go index c8f83f450d518..947ed95fe0598 100644 --- a/src/go/rpk/pkg/cli/cmd/root.go +++ b/src/go/rpk/pkg/cli/cmd/root.go @@ -37,7 +37,6 @@ import ( func Execute() { verbose := false fs := afero.NewOsFs() - mgr := config.NewManager(fs) if !term.IsTerminal(int(os.Stdout.Fd())) { color.NoColor = true @@ -77,7 +76,7 @@ func Execute() { plugincmd.NewCommand(fs), ) - addPlatformDependentCmds(fs, mgr, root) + addPlatformDependentCmds(fs, root) // To support autocompletion even for plugins, we list all plugins now // and add tiny commands to our root command. Cobra works by creating diff --git a/src/go/rpk/pkg/cli/cmd/root_darwin.go b/src/go/rpk/pkg/cli/cmd/root_darwin.go index c2ad57b703d78..c16d212d62781 100644 --- a/src/go/rpk/pkg/cli/cmd/root_darwin.go +++ b/src/go/rpk/pkg/cli/cmd/root_darwin.go @@ -10,13 +10,10 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { cmd.AddCommand(NewRedpandaDarwinCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/root_linux.go b/src/go/rpk/pkg/cli/cmd/root_linux.go index c71ca1b26a9d7..8b66245fddf49 100644 --- a/src/go/rpk/pkg/cli/cmd/root_linux.go +++ b/src/go/rpk/pkg/cli/cmd/root_linux.go @@ -10,23 +10,20 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { - cmd.AddCommand(NewRedpandaCommand(fs, mgr, redpanda.NewLauncher())) +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { + cmd.AddCommand(NewRedpandaCommand(fs, redpanda.NewLauncher())) cmd.AddCommand(NewTuneCommand(fs)) cmd.AddCommand(NewCheckCommand(fs)) cmd.AddCommand(NewIoTuneCmd(fs)) - cmd.AddCommand(NewStartCommand(fs, mgr, redpanda.NewLauncher())) - cmd.AddCommand(NewStopCommand(fs, mgr)) - cmd.AddCommand(NewConfigCommand(fs, mgr)) + cmd.AddCommand(NewStartCommand(fs, redpanda.NewLauncher())) + cmd.AddCommand(NewStopCommand(fs)) + cmd.AddCommand(NewConfigCommand(fs)) cmd.AddCommand(NewStatusCommand(fs)) - cmd.AddCommand(NewModeCommand(mgr)) + cmd.AddCommand(NewModeCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/start.go b/src/go/rpk/pkg/cli/cmd/start.go index db83e7f97b6d4..dfe0251863aeb 100644 --- a/src/go/rpk/pkg/cli/cmd/start.go +++ b/src/go/rpk/pkg/cli/cmd/start.go @@ -15,17 +15,14 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { return common.Deprecated( - redpanda.NewStartCommand(fs, mgr, launcher), + redpanda.NewStartCommand(fs, launcher), "rpk redpanda start", ) } diff --git a/src/go/rpk/pkg/cli/cmd/stop.go b/src/go/rpk/pkg/cli/cmd/stop.go index 86adf9d278e08..7485c774adf0c 100644 --- a/src/go/rpk/pkg/cli/cmd/stop.go +++ b/src/go/rpk/pkg/cli/cmd/stop.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewStopCommand(fs, mgr), + redpanda.NewStopCommand(fs), "rpk redpanda stop", ) } diff --git a/src/go/rpk/pkg/config/config.go b/src/go/rpk/pkg/config/config.go index 5279fd1ea8c1f..0881da098d619 100644 --- a/src/go/rpk/pkg/config/config.go +++ b/src/go/rpk/pkg/config/config.go @@ -10,16 +10,8 @@ package config import ( - "errors" "fmt" - fp "path/filepath" "strings" - - "github.com/mitchellh/mapstructure" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" ) const ( @@ -35,107 +27,35 @@ const ( DefaultBallastFileSize = "1GiB" ) -func InitViper(fs afero.Fs) *viper.Viper { - v := viper.New() - v.SetFs(fs) - v.SetConfigName("redpanda") - v.SetConfigType("yaml") - - // Viper does not take into account our explicit SetConfigType when - // calling ReadInConfig, instead it internally uses SupportedExts. - // Since we only ever want to load yaml, setting this global disables - // ReadInConfig from using any existing json files. - viper.SupportedExts = []string{"yaml"} - - setDefaults(v) - return v -} - -func addConfigPaths(v *viper.Viper) { - v.AddConfigPath("$HOME") - v.AddConfigPath(fp.Join("etc", "redpanda")) - v.AddConfigPath(".") -} - -func setDefaults(v *viper.Viper) { - var traverse func(tree map[string]interface{}, path ...string) - traverse = func(tree map[string]interface{}, path ...string) { - for key, val := range tree { - if subtree, ok := val.(map[string]interface{}); ok { - traverse(subtree, append(path, key)...) - } else { - v.SetDefault( - strings.Join(append(path, key), "."), - val, - ) - } - } - } - traverse(defaultMap()) -} - func Default() *Config { - conf := &Config{} - err := mapstructure.Decode(defaultMap(), conf) - if err != nil { - panic(err) - } - return conf -} - -func defaultMap() map[string]interface{} { - var defaultListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9092, - } - defaultListeners := []interface{}{defaultListener} - var defaultAdminListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9644, - } - defaultAdminListeners := []interface{}{defaultAdminListener} - return map[string]interface{}{ - "config_file": "/etc/redpanda/redpanda.yaml", - "pandaproxy": Pandaproxy{}, - "schema_registry": SchemaRegistry{}, - "redpanda": map[string]interface{}{ - "data_directory": "/var/lib/redpanda/data", - "rpc_server": map[string]interface{}{ - "address": "0.0.0.0", - "port": 33145, + return &Config{ + ConfigFile: "/etc/redpanda/redpanda.yaml", + Redpanda: RedpandaConfig{ + Directory: "/var/lib/redpanda/data", + RPCServer: SocketAddress{ + Address: "0.0.0.0", + Port: 33145, }, - "kafka_api": defaultListeners, - "admin": defaultAdminListeners, - "node_id": 0, - "seed_servers": []interface{}{}, - "developer_mode": true, + KafkaAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9092, + }}, + AdminAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9644, + }}, + SeedServers: []SeedServer{}, + DeveloperMode: true, }, - "rpk": map[string]interface{}{ - "coredump_dir": "/var/lib/redpanda/coredump", + Rpk: RpkConfig{ + CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } } -func findBackup(fs afero.Fs, dir string) (string, error) { - exists, err := afero.Exists(fs, dir) - if err != nil { - return "", err - } - if !exists { - return "", nil - } - files, err := afero.ReadDir(fs, dir) - if err != nil { - return "", err - } - for _, f := range files { - if strings.HasSuffix(f.Name(), ".bk") { - return fmt.Sprintf("%s/%s", dir, f.Name()), nil - } - } - return "", nil -} - func SetMode(mode string, conf *Config) (*Config, error) { m, err := NormalizeMode(mode) if err != nil { @@ -223,131 +143,75 @@ func AvailableModes() []string { } } -func Check(conf *Config) (bool, []error) { - configMap, err := toMap(conf) - if err != nil { - return false, []error{err} - } - - v := viper.New() - err = v.MergeConfigMap(configMap) - if err != nil { - return false, []error{err} - } - return check(v) -} - -func check(v *viper.Viper) (bool, []error) { - errs := checkRedpandaConfig(v) +// Check checks if the redpanda and rpk configuration is valid before running +// the tuners. See: redpanda_checkers. +func (c *Config) Check() (bool, []error) { + errs := checkRedpandaConfig(c) errs = append( errs, - checkRpkConfig(v)..., + checkRpkConfig(c)..., ) ok := len(errs) == 0 return ok, errs } -func checkRedpandaConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetString("redpanda.data_directory") == "" { +func checkRedpandaConfig(cfg *Config) []error { + var errs []error + rp := cfg.Redpanda + // top level check + if rp.Directory == "" { errs = append(errs, fmt.Errorf("redpanda.data_directory can't be empty")) } - if v.GetInt("redpanda.node_id") < 0 { + if rp.ID < 0 { errs = append(errs, fmt.Errorf("redpanda.node_id can't be a negative integer")) } - rpcServerKey := "redpanda.rpc_server" - exists := v.Sub(rpcServerKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", rpcServerKey), - ) + // rpc server + if rp.RPCServer == (SocketAddress{}) { + errs = append(errs, fmt.Errorf("redpanda.rpc_server missing")) } else { - socket := &SocketAddress{} - err := unmarshalKey(v, rpcServerKey, socket) - if err != nil { - errs = append( - errs, - fmt.Errorf("invalid structure for %s", rpcServerKey), - ) - } else { - errs = append( - errs, - checkSocketAddress(*socket, rpcServerKey)..., - ) + saErrs := checkSocketAddress(rp.RPCServer, "redpanda.rpc_server") + if len(saErrs) > 0 { + errs = append(errs, saErrs...) } } - kafkaAPIKey := "redpanda.kafka_api" - exists = v.Get(kafkaAPIKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", kafkaAPIKey), - ) + // kafka api + if len(rp.KafkaAPI) == 0 { + errs = append(errs, fmt.Errorf("redpanda.kafka_api missing")) } else { - var kafkaListeners []NamedSocketAddress - err := unmarshalKey(v, "redpanda.kafka_api", &kafkaListeners) - if err != nil { - log.Error(err) - err = fmt.Errorf( - "%s doesn't have the expected structure", - kafkaAPIKey, - ) - return append( - errs, - err, - ) - } - for i, addr := range kafkaListeners { - configPath := fmt.Sprintf( - "%s.%d", - kafkaAPIKey, - i, - ) - errs = append( - errs, - checkSocketAddress( - SocketAddress{addr.Address, addr.Port}, - configPath, - )..., - ) + for i, addr := range rp.KafkaAPI { + configPath := fmt.Sprintf("redpanda.kafka_api[%d]", i) + saErrs := checkSocketAddress(SocketAddress{addr.Address, addr.Port}, configPath) + if len(saErrs) > 0 { + errs = append(errs, saErrs...) + } } } - var seedServersSlice []*SeedServer // map[string]interface{} - err := unmarshalKey(v, "redpanda.seed_servers", &seedServersSlice) - if err != nil { - log.Error(err) - msg := "redpanda.seed_servers doesn't have the expected structure" - return append( - errs, - errors.New(msg), - ) - } - if len(seedServersSlice) > 0 { - seedServersPath := "redpanda.seed_servers" - for i, seed := range seedServersSlice { - configPath := fmt.Sprintf( - "%s.%d.host", - seedServersPath, - i, - ) - errs = append( - errs, - checkSocketAddress( - seed.Host, - configPath, - )..., - ) + // seed servers + if len(rp.SeedServers) > 0 { + for i, seed := range rp.SeedServers { + configPath := fmt.Sprintf("redpanda.seed_servers[%d].host", i) + saErrs := checkSocketAddress(seed.Host, configPath) + if len(saErrs) > 0 { + errs = append(errs, saErrs...) + } } } return errs } +func checkRpkConfig(cfg *Config) []error { + var errs []error + if cfg.Rpk.TuneCoredump && cfg.Rpk.CoredumpDir == "" { + errs = append(errs, fmt.Errorf("if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty")) + } + return errs +} + func checkSocketAddress(s SocketAddress, configPath string) []error { - errs := []error{} + var errs []error if s.Port == 0 { errs = append(errs, fmt.Errorf("%s.port can't be 0", configPath)) } @@ -356,62 +220,3 @@ func checkSocketAddress(s SocketAddress, configPath string) []error { } return errs } - -func checkRpkConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetBool("rpk.tune_coredump") && v.GetString("rpk.coredump_dir") == "" { - msg := "if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty" - errs = append(errs, errors.New(msg)) - } - return errs -} - -func decoderConfig() mapstructure.DecoderConfig { - return mapstructure.DecoderConfig{ - // Sometimes viper will save int values as strings (i.e. - // through BindPFlag) so we have to allow mapstructure - // to cast them. - WeaklyTypedInput: true, - DecodeHook: mapstructure.ComposeDecodeHookFunc( - // These 2 hooks are viper's default hooks. - // https://github.com/spf13/viper/blob/fb4eafdd9775508c450b90b1b72affeef4a68cf5/viper.go#L1004-L1005 - // They're set here because when decoderConfigOptions' resulting - // viper.DecoderConfigOption is used, viper's hooks are overridden. - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - // This hook translates the pre-21.1.4 configuration format to the - // latest one (see schema.go) - v21_1_4MapToNamedSocketAddressSlice, - // This hook translates the pre-21.4.1 TLS configuration format to the - // latest one (see schema.go) - v21_4_1TlsMapToNamedTlsSlice, - ), - } -} - -func decoderConfigOptions() viper.DecoderConfigOption { - return func(c *mapstructure.DecoderConfig) { - cfg := decoderConfig() - c.DecodeHook = cfg.DecodeHook - c.WeaklyTypedInput = cfg.WeaklyTypedInput - } -} - -func unmarshalKey(v *viper.Viper, key string, val interface{}) error { - return v.UnmarshalKey( - key, - val, - decoderConfigOptions(), - ) -} - -func toMap(conf *Config) (map[string]interface{}, error) { - mapConf := make(map[string]interface{}) - bs, err := yaml.Marshal(conf) - if err != nil { - return nil, err - } - err = yaml.Unmarshal(bs, &mapConf) - return mapConf, err -} diff --git a/src/go/rpk/pkg/config/config_test.go b/src/go/rpk/pkg/config/config_test.go index 6d7d891090bab..5dc21e277a7e4 100644 --- a/src/go/rpk/pkg/config/config_test.go +++ b/src/go/rpk/pkg/config/config_test.go @@ -10,25 +10,18 @@ package config import ( - "path/filepath" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - vyaml "github.com/redpanda-data/redpanda/src/go/rpk/pkg/yaml" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func getValidConfig() *Config { conf := Default() conf.Redpanda.SeedServers = []SeedServer{ - { - SocketAddress{"127.0.0.1", 33145}, - }, - { - SocketAddress{"127.0.0.1", 33146}, - }, + {SocketAddress{"127.0.0.1", 33145}}, + {SocketAddress{"127.0.0.1", 33146}}, } conf.Redpanda.DeveloperMode = false conf.Rpk = RpkConfig{ @@ -59,94 +52,129 @@ func TestSet(t *testing.T) { key string value string format string - check func(st *testing.T, c *Config, mgr *manager) + check func(st *testing.T, c *Config) expectErr bool }{ { - name: "it should parse '1' as an int and not as bool (true)", - key: "redpanda.node_id", - value: "1", - format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + name: "parse '1' as an int and not as bool (true)", + key: "redpanda.node_id", + value: "1", + check: func(st *testing.T, c *Config) { require.Exactly(st, 1, c.Redpanda.ID) }, }, { - name: "it should set single integer fields", - key: "redpanda.node_id", - value: "54312", - format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + name: "set single integer fields", + key: "redpanda.node_id", + value: "54312", + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should detect single integer fields if format isn't passed", + name: "detect single integer fields if format isn't passed", key: "redpanda.node_id", value: "54312", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should set single float fields", - key: "redpanda.float_field", - value: "42.3", - format: "single", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + name: "set single float fields", + key: "redpanda.float_field", + value: "42.3", + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should detect single float fields if format isn't passed", + name: "detect single float fields if format isn't passed", key: "redpanda.float_field", value: "42.3", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should set single string fields", - key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", - format: "single", - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "'/var/lib/differentdir'", c.Redpanda.Directory) + name: "set single string fields", + key: "redpanda.data_directory", + value: "/var/lib/differentdir", + check: func(st *testing.T, c *Config) { + require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should detect single string fields if format isn't passed", + name: "detect single string fields if format isn't passed", key: "redpanda.data_directory", value: "/var/lib/differentdir", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should set single bool fields", - key: "rpk.enable_usage_stats", - value: "true", - format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + name: "set single bool fields", + key: "rpk.enable_usage_stats", + value: "true", + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should detect single bool fields if format isn't passed", + name: "set single bool fields in Other fields (json)", + key: "redpanda.enable_metrics_test", + value: "true", + format: "json", + check: func(st *testing.T, c *Config) { + require.Exactly(st, true, c.Redpanda.Other["enable_metrics_test"]) + }, + }, + { + name: "set single number in Other fields (json)", + key: "redpanda.timeout_test", + value: "123991", + format: "json", + check: func(st *testing.T, c *Config) { + // json.unmarshal stores numbers as float 64. + // see https://pkg.go.dev/encoding/json#Unmarshal + require.Exactly(st, float64(123991), c.Redpanda.Other["timeout_test"]) + }, + }, + { + name: "set single strings in Other fields (json)", + key: "redpanda.test_name", + value: `"my_name"`, + format: "json", + check: func(st *testing.T, c *Config) { + require.Exactly(st, "my_name", c.Redpanda.Other["test_name"]) + }, + }, + { + name: "set objects in Other fields (json)", + key: "redpanda.my_object", + value: `{"name":"test","enabled":true}`, + format: "json", + check: func(st *testing.T, c *Config) { + require.Exactly(st, map[string]interface{}{ + "name": "test", + "enabled": true, + }, c.Redpanda.Other["my_object"]) + }, + }, + { + name: "detect single bool fields if format isn't passed", key: "rpk.enable_usage_stats", value: "true", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, + name: "partially set map fields (yaml)", + key: "rpk", + value: `tune_disk_irq: true +tune_cpu: true`, format: "yaml", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := RpkConfig{ EnableUsageStats: false, Overprovisioned: false, @@ -154,7 +182,7 @@ func TestSet(t *testing.T) { TuneDiskScheduler: false, TuneNomerges: false, TuneDiskIrq: true, - TuneCPU: false, + TuneCPU: true, TuneAioEvents: false, TuneClocksource: false, TuneSwappiness: false, @@ -163,21 +191,20 @@ func TestSet(t *testing.T) { TuneFstrim: false, TuneCoredump: false, TuneDiskWriteCache: false, - CoredumpDir: "/var/lib/redpanda/coredump", } require.Exactly(st, expected, c.Rpk) }, }, { - name: "it should detect pandaproxy client single field if format isn't passed", + name: "detect pandaproxy client single field if format isn't passed", key: "pandaproxy_client.retries", value: "42", - check: func(st *testing.T, c *Config, mgr *manager) { - require.Exactly(st, 42.0, c.PandaproxyClient.Other["retries"]) + check: func(st *testing.T, c *Config) { + require.Exactly(st, 42, c.PandaproxyClient.Other["retries"]) }, }, { - name: "it should detect yaml-formatted values if format isn't passed", + name: "detect yaml-formatted values if format isn't passed", key: "redpanda.kafka_api", value: `- name: external address: 192.168.73.45 @@ -186,7 +213,7 @@ func TestSet(t *testing.T) { address: 10.21.34.58 port: 9092 `, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Name: "external", Address: "192.168.73.45", @@ -200,14 +227,14 @@ func TestSet(t *testing.T) { }, }, { - name: "it should partially set map fields (json)", + name: "partially set map fields (json)", key: "redpanda.kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, format: "json", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -216,13 +243,13 @@ func TestSet(t *testing.T) { }, }, { - name: "it should detect json-formatted values if format isn't passed", + name: "detect json-formatted values if format isn't passed", key: "redpanda.advertised_kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -231,14 +258,22 @@ func TestSet(t *testing.T) { }, }, { - name: "it should fail if the value isn't well formatted (json)", + name: "set value of a slice", + key: "redpanda.admin.port", + value: "9641", + check: func(st *testing.T, c *Config) { + require.Exactly(st, 9641, c.Redpanda.AdminAPI[0].Port) + }, + }, + { + name: "fail if the value isn't well formatted (json)", key: "redpanda", value: `{"seed_servers": []`, format: "json", expectErr: true, }, { - name: "it should fail if the value isn't well formatted (yaml)", + name: "fail if the value isn't well formatted (yaml)", key: "redpanda", value: `seed_servers: - host: @@ -247,96 +282,38 @@ func TestSet(t *testing.T) { expectErr: true, }, { - name: "it should fail if the format isn't supported", + name: "fail if the format isn't supported", key: "redpanda", value: "node_id=1", format: "toml", expectErr: true, }, { - name: "it should fail if no key is passed", + name: "fail if no key is passed", value: `node_id=1`, expectErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set(tt.key, tt.value, tt.format) - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) - } - }) - } -} - -func TestMerge(t *testing.T) { - tests := []struct { - name string - config Config - check func(st *testing.T, c *Config, mgr *manager) - expectErr bool - }{ - { - name: "it should merge Kafka API spec", - config: Config{ - Redpanda: RedpandaConfig{ - KafkaAPI: []NamedSocketAddress{{ - Name: "kafka-api-name", - Address: "1.2.3.4", - Port: 9123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "kafka-api-name", c.Redpanda.KafkaAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Redpanda.KafkaAPI[0].Address) - require.Exactly(st, 9123, c.Redpanda.KafkaAPI[0].Port) - }, - }, { - name: "it should merge Pandaproxy API spec", - config: Config{ - Pandaproxy: &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{{ - Name: "proxy-api-name", - Address: "1.2.3.4", - Port: 8123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "proxy-api-name", c.Pandaproxy.PandaproxyAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Pandaproxy.PandaproxyAPI[0].Address) - require.Exactly(st, 8123, c.Pandaproxy.PandaproxyAPI[0].Port) - }, + name: "fail if deep unrecognized value is passed", + key: "redpanda.unrecognized.name", + value: "foo", + expectErr: true, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Merge(&tt.config) + cfg, err := new(Params).Load(fs) + require.NoError(t, err) + err = cfg.Set(tt.key, tt.value, tt.format) if tt.expectErr { require.Error(t, err) return } require.NoError(t, err) if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) + tt.check(t, cfg) } }) } @@ -370,53 +347,6 @@ func TestDefault(t *testing.T) { require.Exactly(t, expected, defaultConfig) } -func TestRead(t *testing.T) { - const baseDir string = "/etc/redpanda" - tests := []struct { - name string - path string - before func(afero.Fs, string) error - want func() *Config - wantErr bool - }{ - { - name: "shall return a config struct filled with values from the file", - before: func(fs afero.Fs, path string) error { - bs, err := yaml.Marshal(getValidConfig()) - if err != nil { - return err - } - if err = fs.MkdirAll(baseDir, 0o755); err != nil { - return err - } - _, err = utils.WriteBytes(fs, bs, path) - return err - }, - path: baseDir + "/redpanda.yaml", - want: getValidConfig, - wantErr: false, - }, - // TODO: Add tests for when the config file has missing objects - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := tt.before(fs, tt.path) - require.NoError(t, err) - got, err := mgr.Read(tt.path) - want := tt.want() - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.True(t, filepath.IsAbs(got.ConfigFile)) - require.Exactly(t, want, got) - }) - } -} - func TestWrite(t *testing.T) { tests := []struct { name string @@ -426,103 +356,54 @@ func TestWrite(t *testing.T) { expected string }{ { - name: "it should write the default values for redpanda.kafka_api if it's null", - conf: getValidConfig, - wantErr: false, + name: "write default values", + conf: getValidConfig, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - host: + address: 127.0.0.1 + port: 33145 + - host: + address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "it should write the default values if redpanda.kafka_api is missing", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without advertised_kafka_api", + name: "write additional values", conf: func() *Config { c := getValidConfig() c.Redpanda.AdvertisedRPCAPI = &SocketAddress{ @@ -531,114 +412,55 @@ schema_registry: {} } return c }, - wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_rpc_api: - address: 174.32.64.2 - port: 33145 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - host: + address: 127.0.0.1 + port: 33145 + - host: + address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + advertised_rpc_api: + address: 174.32.64.2 + port: 33145 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file without advertised_rpc_api", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdvertisedKafkaAPI = []NamedSocketAddress{{ - Address: "174.32.64.2", - Port: 9092, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_kafka_api: - - address: 174.32.64.2 - port: 9092 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without an rpk config object", + name: "write if empty struct is passed", conf: func() *Config { c := getValidConfig() c.Rpk = RpkConfig{} @@ -646,393 +468,49 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false -schema_registry: {} -`, - }, - { - name: "shall fail with an invalid config", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.Directory = "" - return c - }, - wantErr: true, - }, - { - name: "shall write a valid config file with an rpk config object", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall leave unrecognized fields untouched", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - node_id: 1 - - host: - address: 127.0.0.1 - port: 33146 - node_id: 2 - target_quota_byte_rate: 1000000 -unrecognized_top_field: - child: true -`, - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -unrecognized_top_field: - child: true -`, - }, - { - name: "should merge the new config onto the current one, not just overwrite it", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: func() *Config { - conf := getValidConfig() - conf.Redpanda.SeedServers = []SeedServer{} - conf.Redpanda.Directory = "/different/path" - return conf - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /different/path - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.KafkaAPI[0].Name = "outside" - c.Redpanda.KafkaAPITLS = []ServerTLS{{ - Name: "outside", - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - Other: map[string]interface{}{"principal_mapping_rules": "DEFAULT"}, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - name: outside - port: 9092 - kafka_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - name: outside - principal_mapping_rules: DEFAULT - require_client_auth: true - truststore_file: /etc/certs/ca.crt - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - host: + address: 127.0.0.1 + port: 33145 + - host: + address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with admin tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdminAPITLS = []ServerTLS{{ - KeyFile: "/etc/certs/admin/cert.key", - TruststoreFile: "/etc/certs/admin/ca.crt", - CertFile: "/etc/certs/admin/cert.crt", - Enabled: true, - RequireClientAuth: true, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: false + tune_network: false + tune_disk_scheduler: false + tune_disk_nomerges: false + tune_disk_write_cache: false + tune_disk_irq: false + tune_fstrim: false + tune_cpu: false + tune_aio_events: false + tune_clocksource: false + tune_swappiness: false + tune_transparent_hugepages: false + enable_memory_locking: false + tune_coredump: false + tune_ballast_file: false + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_tls: - - cert_file: /etc/certs/admin/cert.crt - enabled: true - key_file: /etc/certs/admin/cert.key - require_client_auth: true - truststore_file: /etc/certs/admin/ca.crt - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write config with log_segment_size configuration", + name: "write unrecognized values ('Other' map).", conf: func() *Config { c := getValidConfig() size := 536870912 @@ -1044,469 +522,47 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - log_segment_size: 536870912 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with full pandaproxy configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - PandaproxyAPITLS: []ServerTLS{{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }}, - AdvertisedPandaproxyAPI: []NamedSocketAddress{ - { - Name: "advertised", - Address: "2.3.4.1", - Port: 2341, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - advertised_pandaproxy_api: - - address: 2.3.4.1 - name: advertised - port: 2341 - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 - pandaproxy_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy api only configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy client configuration", - conf: func() *Config { - c := getValidConfig() - c.PandaproxyClient = &KafkaClient{ - Brokers: []SocketAddress{ - { - Address: "1.2.3.4", - Port: 1234, - }, - }, - BrokerTLS: ServerTLS{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -pandaproxy_client: - broker_tls: - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt - brokers: - - address: 1.2.3.4 - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandproxy client SASL mechanism and SCRAM credentials", - conf: func() *Config { - c := getValidConfig() - mechanism := "abc" - username := "user" - password := "pass" - c.PandaproxyClient = &KafkaClient{ - SASLMechanism: &mechanism, - SCRAMUsername: &username, - SCRAMPassword: &password, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -pandaproxy_client: - sasl_mechanism: abc - scram_password: pass - scram_username: user -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file with scram configured", - conf: func() *Config { - c := getValidConfig() - c.Rpk.KafkaAPI.SASL = &SASL{ - User: "scram_user", - Password: "scram_password", - Mechanism: "SCRAM-SHA-256", - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - host: + address: 127.0.0.1 + port: 33145 + - host: + address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false + log_segment_size: 536870912 rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - kafka_api: - sasl: - password: scram_password - type: SCRAM-SHA-256 - user: scram_user - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "should update an existing config with single kafka_api & advertised_kafka_api obj to a list", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - address: 0.0.0.0 - port: 9092 - advertised_kafka_api: - address: 1.cluster.redpanda.io - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: Default, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - advertised_kafka_api: - - address: 1.cluster.redpanda.io - port: 9092 - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false schema_registry: {} `, }, @@ -1515,19 +571,12 @@ schema_registry: {} t.Run(tt.name, func(t *testing.T) { path := tt.conf().ConfigFile fs := afero.NewMemMapFs() - mgr := NewManager(fs) if tt.existingConf != "" { _, err := utils.WriteBytes(fs, []byte(tt.existingConf), path) require.NoError(t, err) - } else { - // Write the config file so that WriteConfig backs it up - // when the new one is written. - err := vyaml.Persist(fs, tt.conf(), path) - require.NoError(t, err) } - _, err := mgr.Read(path) - require.NoError(t, err) - err = mgr.Write(tt.conf()) + + err := tt.conf().Write(fs) if tt.wantErr { require.Error(t, err) return @@ -1538,96 +587,6 @@ schema_registry: {} require.NoError(t, err) content := string(contentBytes) require.Equal(t, tt.expected, content) - backup, err := findBackup(fs, filepath.Dir(path)) - require.NoError(t, err) - _, err = fs.Stat(backup) - require.NoError(t, err) - }) - } -} - -func TestWriteLoaded(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set( - "rpk.admin", - `[{"address": "192.168.54.2","port": 9092}]`, - "json", - ) - require.NoError(t, err) - - mgr.WriteLoaded() - conf, err := mgr.Get() - require.NoError(t, err) - - newMgr := NewManager(fs) - newConf, err := newMgr.Read(Default().ConfigFile) - require.NoError(t, err) - - require.Exactly(t, conf, newConf) -} - -func TestReadOrGenerate(t *testing.T) { - tests := []struct { - name string - setup func(afero.Fs) error - configFile string - check func(st *testing.T, conf *Config) - expectError bool - }{ - { - name: "it should generate a config file at the given location", - configFile: Default().ConfigFile, - }, - { - name: "it shouldn't fail if there's a config file already", - setup: func(fs afero.Fs) error { - conf := Default() - mgr := NewManager(fs) - return mgr.Write(conf) - }, - configFile: Default().ConfigFile, - }, - { - name: "it should fail if the existing config file's content isn't valid yaml", - setup: func(fs afero.Fs) error { - bs := []byte(`redpanda: -- something`) - return vyaml.Persist(fs, bs, Default().ConfigFile) - }, - configFile: Default().ConfigFile, - expectError: true, - }, - { - name: "it should set config_file to the right value", - configFile: "./redpanda.yaml", - check: func(st *testing.T, conf *Config) { - path, err := filepath.Abs("./redpanda.yaml") - require.NoError(st, err) - require.Exactly(st, conf.ConfigFile, path) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - if tt.setup != nil { - err := tt.setup(fs) - require.NoError(t, err) - } - _, err := readOrGenerate(fs, InitViper(fs), tt.configFile) - if tt.expectError { - require.Error(t, err) - return - } - require.NoError(t, err) - conf, err := mgr.Read(tt.configFile) - require.NoError(t, err) - if tt.check != nil { - tt.check(t, conf) - } }) } } @@ -1801,7 +760,7 @@ func TestCheckConfig(t *testing.T) { c.Redpanda.KafkaAPI[0].Port = 0 return c }, - expected: []string{"redpanda.kafka_api.0.port can't be 0"}, + expected: []string{"redpanda.kafka_api[0].port can't be 0"}, }, { name: "shall return an error when the Kafka API address is empty", @@ -1810,7 +769,7 @@ func TestCheckConfig(t *testing.T) { c.Redpanda.KafkaAPI[0].Address = "" return c }, - expected: []string{"redpanda.kafka_api.0.address can't be empty"}, + expected: []string{"redpanda.kafka_api[0].address can't be empty"}, }, { name: "shall return an error when one of the seed servers' address is empty", @@ -1819,7 +778,7 @@ func TestCheckConfig(t *testing.T) { c.Redpanda.SeedServers[0].Host.Address = "" return c }, - expected: []string{"redpanda.seed_servers.0.host.address can't be empty"}, + expected: []string{"redpanda.seed_servers[0].host.address can't be empty"}, }, { name: "shall return an error when one of the seed servers' port is 0", @@ -1828,7 +787,7 @@ func TestCheckConfig(t *testing.T) { c.Redpanda.SeedServers[1].Host.Port = 0 return c }, - expected: []string{"redpanda.seed_servers.1.host.port can't be 0"}, + expected: []string{"redpanda.seed_servers[1].host.port can't be 0"}, }, { name: "shall return no errors when tune_coredump is set to false," + @@ -1849,8 +808,7 @@ func TestCheckConfig(t *testing.T) { c.Rpk.CoredumpDir = "" return c }, - expected: []string{"if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty"}, + expected: []string{"if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty"}, }, { name: "shall return no error if setup is empty," + @@ -1865,7 +823,7 @@ func TestCheckConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, got := Check(tt.conf()) + _, got := tt.conf().Check() errMsgs := []string{} for _, err := range got { errMsgs = append(errMsgs, err.Error()) @@ -1874,31 +832,3 @@ func TestCheckConfig(t *testing.T) { }) } } - -func TestWriteAndGenerateNodeUuid(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - baseDir := "/etc/redpanda" - path := baseDir + "/redpanda.yaml" - conf := getValidConfig() - conf.ConfigFile = path - bs, err := yaml.Marshal(conf) - require.NoError(t, err) - err = fs.MkdirAll(baseDir, 0o755) - require.NoError(t, err) - _, err = utils.WriteBytes(fs, bs, path) - require.NoError(t, err) - err = mgr.WriteNodeUUID(conf) - require.NoError(t, err) - require.NotEqual(t, "", conf.NodeUUID) - readConf, err := mgr.Read(path) - require.NoError(t, err) - require.Exactly(t, conf, readConf) -} - -func TestGet(t *testing.T) { - mgr := NewManager(afero.NewMemMapFs()) - conf, err := mgr.Get() - require.NoError(t, err) - require.Exactly(t, Default(), conf) -} diff --git a/src/go/rpk/pkg/config/find.go b/src/go/rpk/pkg/config/find.go deleted file mode 100644 index 4b2147b72b3f6..0000000000000 --- a/src/go/rpk/pkg/config/find.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" -) - -func FindConfigFile(fs afero.Fs) (string, error) { - log.Debugf("Looking for the redpanda config file") - configPathProviders := []func() ([]string, error){ - currentDirectory, - sysConfDirectory, - homeDirectory, - } - - lookedUpPaths := []string{} - for _, provider := range configPathProviders { - paths, err := provider() - if err != nil { - return "", err - } - for _, path := range paths { - candidate := filepath.Join(path, "redpanda.yaml") - lookedUpPaths = append(lookedUpPaths, candidate) - log.Debugf("Looking for redpanda config file in '%s'", path) - if exists, _ := afero.Exists(fs, candidate); exists { - return candidate, nil - } - } - } - // os.PathError can be checked with os.IsNotExist. - return "", &os.PathError{ - Op: "Open", - Path: strings.Join(lookedUpPaths, ", "), - Err: os.ErrNotExist, - } -} - -func sysConfDirectory() ([]string, error) { - return []string{"/etc/redpanda"}, nil -} - -func currentDirectory() ([]string, error) { - currentDir, err := os.Getwd() - if err != nil { - return nil, err - } - return []string{currentDir}, nil -} - -func homeDirectory() ([]string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, err - } - return []string{homeDir}, nil -} diff --git a/src/go/rpk/pkg/config/find_test.go b/src/go/rpk/pkg/config/find_test.go deleted file mode 100644 index bf6dbeb730582..0000000000000 --- a/src/go/rpk/pkg/config/find_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestFindConfig(t *testing.T) { - tests := []struct { - name string - before func(fs afero.Fs) - want string - wantErr bool - }{ - { - name: "should return an error when config is not found", - before: func(afero.Fs) {}, - want: "", - wantErr: true, - }, - { - name: "should return config file from home directory", - before: func(fs afero.Fs) { - createConfigIn(fs, homeDir()) - }, - want: filepath.Join(homeDir(), "redpanda.yaml"), - }, - { - name: "should return config file from 'etc' directory", - before: func(fs afero.Fs) { - createConfigIn(fs, filepath.Join("/", "etc", "redpanda")) - }, - want: filepath.Join("/", "etc", "redpanda", "redpanda.yaml"), - }, - { - name: "should return config file from current directory", - before: func(fs afero.Fs) { - createConfigIn(fs, currentDir()) - }, - want: filepath.Join(currentDir(), "redpanda.yaml"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - tt.before(fs) - got, err := FindConfigFile(fs) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, tt.want, got) - }) - } -} - -func createConfigIn(fs afero.Fs, path string) { - fs.Create(filepath.Join(path, "redpanda.yaml")) -} - -func currentDir() string { - d, _ := os.Getwd() - return d -} - -func homeDir() string { - d, _ := os.UserHomeDir() - return d -} diff --git a/src/go/rpk/pkg/config/license.go b/src/go/rpk/pkg/config/license.go deleted file mode 100644 index 5ecf38d923de0..0000000000000 --- a/src/go/rpk/pkg/config/license.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "hash/crc32" - "time" - - log "github.com/sirupsen/logrus" -) - -type LicenseKey struct { - Organization string `json:"o"` - ExpirationYear uint16 `json:"y"` - ExpirationMonth uint8 `json:"m"` - ExpirationDay uint8 `json:"d"` - Checksum uint32 `json:"c"` -} - -// Checks the key and prints a warning if the key is invalid. -func CheckAndPrintNotice(key string) { - err := CheckLicenseKey(key) - if err != nil { - log.Info(err) - return - } -} - -// Checks the license key, which is a Base64-encoded JSON object with the the -// organization it was granted to, the expiration date, and the hash of the -// concatenation of those fields. -// Returns true if the exp. date hasn't passed yet. Returns false otherwise. -// Returns an error if the key isn't base64-encoded or if it's not valid JSON. -func CheckLicenseKey(key string) error { - msg := "Please get a new one at https://vectorized.io/download-trial/" - if key == "" { - return errors.New("Missing license key. " + msg) - } - decoded, err := base64.StdEncoding.DecodeString(key) - invalidErr := errors.New("Invalid license key. " + msg) - if err != nil { - return invalidErr - } - lk := &LicenseKey{} - err = json.Unmarshal(decoded, lk) - if err != nil { - return invalidErr - } - - content := fmt.Sprintf( - "%s%d%d%d", - lk.Organization, - lk.ExpirationYear, - lk.ExpirationMonth, - lk.ExpirationDay, - ) - - checksum := crc32.ChecksumIEEE([]byte(content)) - - if lk.Checksum != checksum { - return invalidErr - } - - expDate := time.Date( - int(lk.ExpirationYear), - time.Month(lk.ExpirationMonth), - int(lk.ExpirationDay), - 0, 0, 0, 0, - time.UTC, - ) - - if time.Now().After(expDate) { - return errors.New("Your license key has expired. " + msg) - } - return nil -} diff --git a/src/go/rpk/pkg/config/license_test.go b/src/go/rpk/pkg/config/license_test.go deleted file mode 100644 index 20b5fd9df80de..0000000000000 --- a/src/go/rpk/pkg/config/license_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config_test - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "hash/crc32" - "testing" - "time" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/stretchr/testify/require" -) - -const vectorizedOrg string = "vectorized.io" - -func makeKey(org string, exp time.Time, overrideHash uint32) (string, error) { - content := fmt.Sprintf( - "%s%d%d%d", - org, - exp.Year(), - exp.Month(), - exp.Day(), - ) - var checksum uint32 - if overrideHash != 0 { - checksum = overrideHash - } else { - checksum = crc32.ChecksumIEEE([]byte(content)) - } - lk := &config.LicenseKey{ - Organization: org, - ExpirationYear: uint16(exp.Year()), - ExpirationMonth: uint8(exp.Month()), - ExpirationDay: uint8(exp.Day()), - Checksum: checksum, - } - bs, err := json.Marshal(lk) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(bs), nil -} - -func TestCheckLicenseKey(t *testing.T) { - msg := "Please get a new one at https://vectorized.io/download-trial/" - tests := []struct { - name string - key func() (string, error) - expected string - }{ - { - name: "key is valid if the hashes match and the expiration date hasn't passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "", - }, - { - name: "key is invalid if the hashes don't match", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 123) - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if the expiration date has already passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(-48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "Your license key has expired. " + msg, - }, - { - name: "key is invalid if it's not base64-encoded", - key: func() (string, error) { - return "this is defs not base 64", nil - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if it's not valid base64-encoded JSON", - key: func() (string, error) { - jsonStr := []byte("{\"thisJson\":\"is invalid\"") - return base64.StdEncoding.EncodeToString(jsonStr), nil - }, - expected: "Invalid license key. " + msg, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(st *testing.T) { - key, err := tt.key() - require.NoError(st, err) - err = config.CheckLicenseKey(key) - if tt.expected == "" { - require.Nil(st, err) - return - } - require.EqualError(st, err, tt.expected) - }) - } -} diff --git a/src/go/rpk/pkg/config/manager.go b/src/go/rpk/pkg/config/manager.go deleted file mode 100644 index a8a098b1fc718..0000000000000 --- a/src/go/rpk/pkg/config/manager.go +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - - "github.com/google/uuid" - "github.com/mitchellh/mapstructure" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" -) - -type Manager interface { - // Reads the config from the given path - Read(path string) (*Config, error) - // Writes the config to Config.ConfigFile - Write(conf *Config) error - // Writes the currently-loaded config to redpanda.config_file - WriteLoaded() error - // Get the currently-loaded config - Get() (*Config, error) - // Sets key to the given value (parsing it according to the format) - Set(key, value, format string) error - // If path is empty, tries to find the file in the default locations. - // Otherwise, it tries to read the file and load it. If the file doesn't - // exist, it tries to create it with the default configuration. - FindOrGenerate(path string) (*Config, error) - // Tries reading a config file at the given path, or tries to find it in - // the default locations if it doesn't exist. - ReadOrFind(path string) (*Config, error) - // Generates and writes the node's UUID - WriteNodeUUID(conf *Config) error - // Merges an input config to the currently-loaded map - Merge(conf *Config) error -} - -type manager struct { - fs afero.Fs - v *viper.Viper -} - -func NewManager(fs afero.Fs) Manager { - return &manager{fs, InitViper(fs)} -} - -func (m *manager) FindOrGenerate(path string) (*Config, error) { - if path == "" { - addConfigPaths(m.v) - err := m.v.ReadInConfig() - if err == nil { - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - if !notFound { - return nil, err - } - path = Default().ConfigFile - } - return readOrGenerate(m.fs, m.v, path) -} - -// Tries reading a config file at the given path, or generates a default config -// and writes it to the path. -func readOrGenerate(fs afero.Fs, v *viper.Viper, path string) (*Config, error) { - abs, err := absPath(path) - if err != nil { - return nil, err - } - v.SetConfigFile(abs) - err = v.ReadInConfig() - if err == nil { - // The config file's there, there's nothing to do. - return unmarshal(v) - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - notExist := os.IsNotExist(err) - if err != nil && !notFound && !notExist { - return nil, fmt.Errorf( - "An error happened while trying to read %s: %v", - abs, - err, - ) - } - log.Debug(err) - log.Infof( - "Couldn't find config file at %s. Generating it.", - abs, - ) - v.Set("config_file", abs) - err = createConfigDir(fs, abs) - if err != nil { - return nil, err - } - err = v.WriteConfigAs(abs) - if err != nil { - return nil, fmt.Errorf( - "Couldn't write config to %s: %v", - abs, - err, - ) - } - return unmarshal(v) -} - -func (m *manager) ReadOrFind(path string) (*Config, error) { - var err error - if path == "" { - path, err = FindConfigFile(m.fs) - if err != nil { - return nil, err - } - } - return m.Read(path) -} - -func (m *manager) Read(path string) (*Config, error) { - // If the path was set, try reading only from there. - abs, err := filepath.Abs(path) - if err != nil { - return nil, err - } - m.v.SetConfigFile(abs) - err = m.v.ReadInConfig() - if err != nil { - return nil, err - } - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err -} - -func (m *manager) WriteNodeUUID(conf *Config) error { - id, err := uuid.NewUUID() - if err != nil { - return err - } - conf.NodeUUID = id.String() - return m.Write(conf) -} - -func (m *manager) Get() (*Config, error) { - return unmarshal(m.v) -} - -// Checks config and writes it to the given path. -func (m *manager) Write(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - // Merge the config into a new viper.Viper instance to prevent - // concurrent writes to the underlying config map. - v := InitViper(m.fs) - current, err := unmarshal(m.v) - if err != nil { - return err - } - currentMap, err := toMap(current) - if err != nil { - return err - } - v.MergeConfigMap(currentMap) - v.MergeConfigMap(confMap) - return checkAndWrite(m.fs, v, conf.ConfigFile) -} - -// Writes the currently loaded config. -func (m *manager) WriteLoaded() error { - return checkAndWrite(m.fs, m.v, m.v.GetString("config_file")) -} - -func write(fs afero.Fs, v *viper.Viper, path string) error { - err := createConfigDir(fs, path) - if err != nil { - return err - } - err = v.WriteConfigAs(path) - if err != nil { - return err - } - log.Debugf( - "Configuration written to %s.", - path, - ) - return nil -} - -func (m *manager) setDeduceFormat(key, value string) error { - replace := func(key string, newValue interface{}) error { - newV := viper.New() - newV.Set(key, newValue) - return m.v.MergeConfigMap(newV.AllSettings()) - } - - var newVal interface{} - switch { - case json.Unmarshal([]byte(value), &newVal) == nil: // Try JSON - return replace(key, newVal) - - case yaml.Unmarshal([]byte(value), &newVal) == nil: // Try YAML - return replace(key, newVal) - - default: // Treat the value as a "single" - m.v.Set(key, parse(value)) - return nil - } -} - -func (m *manager) Set(key, value, format string) error { - if key == "" { - return errors.New("empty config field key") - } - if format == "" { - return m.setDeduceFormat(key, value) - } - var newConfValue interface{} - switch strings.ToLower(format) { - case "single": - m.v.Set(key, parse(value)) - return nil - case "yaml": - err := yaml.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - case "json": - err := json.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - default: - return fmt.Errorf("unsupported format %s", format) - } - newV := viper.New() - newV.Set(key, newConfValue) - return m.v.MergeConfigMap(newV.AllSettings()) -} - -func (m *manager) Merge(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - return m.v.MergeConfigMap(confMap) -} - -func checkAndWrite(fs afero.Fs, v *viper.Viper, path string) error { - ok, errs := check(v) - if !ok { - reasons := []string{} - for _, err := range errs { - reasons = append(reasons, err.Error()) - } - return errors.New(strings.Join(reasons, ", ")) - } - lastBackupFile, err := findBackup(fs, filepath.Dir(path)) - if err != nil { - return err - } - exists, err := afero.Exists(fs, path) - if err != nil { - return err - } - if !exists { - // If the config doesn't exist, just write it. - return write(fs, v, path) - } - // Otherwise, backup the current config file, write the new one, and - // try to recover if there's an error. - log.Debug("Backing up the current config") - backup, err := utils.BackupFile(fs, path) - if err != nil { - return err - } - log.Debugf("Backed up the current config to %s", backup) - if lastBackupFile != "" && lastBackupFile != backup { - log.Debug("Removing previous backup file") - err = fs.Remove(lastBackupFile) - if err != nil { - return err - } - } - log.Debugf("Writing the new redpanda config to '%s'", path) - err = write(fs, v, path) - if err != nil { - return recover(fs, backup, path, err) //nolint:revive // false positive: this recover function is different from built-in recover - } - return nil -} - -func recover(fs afero.Fs, backup, path string, err error) error { - log.Infof("Recovering the previous confing from %s", backup) - recErr := utils.CopyFile(fs, backup, path) - if recErr != nil { - msg := "couldn't persist the new config due to '%v'," + - " nor recover the backup due to '%v" - return fmt.Errorf(msg, err, recErr) - } - return fmt.Errorf("couldn't persist the new config due to '%v'", err) -} - -func unmarshal(v *viper.Viper) (*Config, error) { - result := &Config{} - decoderConfig := decoderConfig() - decoderConfig.Result = result - decoder, err := mapstructure.NewDecoder(&decoderConfig) - if err != nil { - return nil, err - } - err = decoder.Decode(v.AllSettings()) - if err != nil { - return nil, err - } - if result.ConfigFile == "" { - result.ConfigFile, err = absPath(v.ConfigFileUsed()) - if err != nil { - return nil, err - } - } - return result, nil -} - -// Redpanda version < 21.1.4 only supported a single anonymous listener and a -// single anonymous advertised address. This custom decode function translates -// a single SocketAddress-equivalent map[string]interface{} into a -// []NamedSocketAddress. -func v21_1_4MapToNamedSocketAddressSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]NamedSocketAddress{}) { - if from.Kind() == reflect.Map { - sa := NamedSocketAddress{} - err := mapstructure.Decode(data, &sa) - if err != nil { - return nil, err - } - return []NamedSocketAddress{sa}, nil - } - } - return data, nil -} - -// Redpanda version <= 21.4.1 only supported a single TLS config. This custom -// decode function translates a single TLS config-equivalent -// map[string]interface{} into a []ServerTLS. -//nolint:revive // using underscore here is intended -func v21_4_1TlsMapToNamedTlsSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]ServerTLS{}) { - if from.Kind() == reflect.Map { - tls := ServerTLS{} - err := mapstructure.Decode(data, &tls) - if err != nil { - return nil, err - } - return []ServerTLS{tls}, nil - } - } - return data, nil -} - -func parse(val string) interface{} { - if i, err := strconv.Atoi(val); err == nil { - return i - } - if f, err := strconv.ParseFloat(val, 64); err == nil { - return f - } - if b, err := strconv.ParseBool(val); err == nil { - return b - } - return val -} - -func absPath(path string) (string, error) { - absPath, err := filepath.Abs(path) - if err != nil { - return "", fmt.Errorf( - "couldn't convert the used config file path to"+ - " absolute: %s", - path, - ) - } - return absPath, nil -} - -func createConfigDir(fs afero.Fs, configFile string) error { - dir := filepath.Dir(configFile) - err := fs.MkdirAll(dir, 0o755) - if err != nil { - return fmt.Errorf( - "couldn't create config dir %s: %v", - dir, - err, - ) - } - return nil -} diff --git a/src/go/rpk/pkg/config/params.go b/src/go/rpk/pkg/config/params.go index ed701b43e7763..0a5cc52271292 100644 --- a/src/go/rpk/pkg/config/params.go +++ b/src/go/rpk/pkg/config/params.go @@ -10,11 +10,13 @@ package config import ( + "encoding/json" "errors" "fmt" "net" "os" "path/filepath" + "reflect" "strconv" "strings" "syscall" @@ -23,7 +25,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // This file contains the Params type, which will eventually be created in @@ -210,7 +212,7 @@ func ParamsFromCommand(cmd *cobra.Command) *Params { } val := f.Value.String() - // Value.String() adds backets to slice types, and we + // Value.String() adds brackets to slice types, and we // need to strip that here. if stripBrackets { if len(val) > 0 && val[0] == '[' && val[len(val)-1] == ']' { @@ -234,8 +236,24 @@ func ParamsFromCommand(cmd *cobra.Command) *Params { // * Sets unset default values. // func (p *Params) Load(fs afero.Fs) (*Config, error) { + cf := "/etc/redpanda/redpanda.yaml" + // If we have a config path loaded (through --config flag) the user + // expect to load or create the file from this directory. + if p.ConfigPath != "" { + if exist, _ := afero.Exists(fs, p.ConfigPath); !exist { + err := fs.MkdirAll(filepath.Dir(p.ConfigPath), 0o755) + if err != nil { + return nil, err + } + } + abs, err := filepath.Abs(p.ConfigPath) + if err != nil { + return nil, err + } + cf = abs + } c := &Config{ - ConfigFile: "/etc/redpanda/redpanda.yaml", + ConfigFile: cf, Redpanda: RedpandaConfig{ Directory: "/var/lib/redpanda/data", RPCServer: SocketAddress{ @@ -255,6 +273,9 @@ func (p *Params) Load(fs afero.Fs) (*Config, error) { Rpk: RpkConfig{ CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } if err := p.readConfig(fs, c); err != nil { @@ -276,9 +297,8 @@ func (p *Params) Load(fs afero.Fs) (*Config, error) { func (c *Config) Write(fs afero.Fs) (rerr error) { cfgPath := c.loadedPath if cfgPath == "" { - cfgPath = Default().ConfigFile + cfgPath = c.ConfigFile } - b, err := yaml.Marshal(c) if err != nil { return fmt.Errorf("marshal error in loaded config, err: %s", err) @@ -378,8 +398,12 @@ func (p *Params) readConfig(fs afero.Fs, c *Config) error { return fmt.Errorf("unable to yaml decode %s: %v", path, err) } yaml.Unmarshal(file, &c.file) // cannot error since previous did not - c.loadedPath = path - + abs, err := filepath.Abs(path) + if err != nil { + return err + } + c.loadedPath = abs + c.ConfigFile = abs return nil } @@ -552,3 +576,133 @@ func (c *Config) addUnsetDefaults() { r.AdminAPI.Addresses = []string{"127.0.0.1:9644"} } } + +// Set allow to set a single configuration property by passing a key value pair +// +// Key: string containing the yaml property tag, e.g: 'rpk.admin_api'. +// Value: string representation of the value, either single value or partial +// representation. +// Format: either json or yaml (default: yaml). +func (c *Config) Set(key, value, format string) error { + if key == "" { + return fmt.Errorf("key field must not be empty") + } + props := strings.Split(key, ".") + rv := reflect.ValueOf(c).Elem() + field, other, err := getField(props, rv) + if err != nil { + return err + } + isOther := other != reflect.Value{} + + if isOther { + field = other + } + + if field.CanAddr() { + i := field.Addr().Interface() + in := value + switch strings.ToLower(format) { + // single is deprecated, leaving it here for backward compatibility. + case "yaml", "single", "": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("%s: %s", p, value) + } + err = yaml.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + case "json": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("{%q: %s}", p, value) + } + err = json.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + default: + return fmt.Errorf("unsupported format %s", format) + } + } + return errors.New("rpk bug, please describe how you encountered this at https://github.com/redpanda-data/redpanda/issues/new?assignees=&labels=kind%2Fbug&template=01_bug_report.md") +} + +// getField deeply search in p for the value that reflect property props. +func getField(props []string, p reflect.Value) (reflect.Value, reflect.Value, error) { + if len(props) == 0 { + return p, reflect.Value{}, nil + } + if p.Kind() == reflect.Slice { + if p.Len() == 0 { + p.Set(reflect.New(p.Type().Elem())) + } else { + p = p.Index(0) + } + } + + // If is a nil pointer we assign the zero value, and we reassign p to the + // value that p points to + if p.Kind() == reflect.Ptr { + if p.IsNil() { + p.Set(reflect.New(p.Type().Elem())) + } + p = reflect.Indirect(p) + } + if p.Kind() == reflect.Struct { + newP, other, err := getFieldByTag(props[0], p) + if err != nil { + return reflect.Value{}, reflect.Value{}, err + } + // if is "Other" map field, we stop the recursion and return + if (other != reflect.Value{}) { + // user may try to set deep unmanaged property: + // rpk.unmanaged.name = "name" + if len(props) > 1 { + return reflect.Value{}, reflect.Value{}, fmt.Errorf("unable to set property %q using rpk", strings.Join(props, ".")) + } + return reflect.Value{}, other, nil + } + return getField(props[1:], newP) + } + return reflect.Value{}, reflect.Value{}, fmt.Errorf("unable to set field of type %v", p.Type()) +} + +// getFieldByTag finds a field with a given yaml tag and returns 3 parameters: +// +// 1. if tag is found within the struct, return the field. +// 2. if tag is not found _but_ the struct has "Other" field, return Other. +// 3. Error if it can't find the given tag and "Other" field is unavailable. +func getFieldByTag(tag string, p reflect.Value) (reflect.Value, reflect.Value, error) { + t := p.Type() + var other bool + // Loop struct to get the field that match tag. + for i := 0; i < p.NumField(); i++ { + // rpk allows blindly setting unknown configuration parameters in + // Other map[string]interface{} fields + if t.Field(i).Name == "Other" { + other = true + } + yt := t.Field(i).Tag.Get("yaml") + + // yaml struct tags can contain flags such as omitempty, + // when tag.Get("yaml") is called it will return + // "my_tag,omitempty" + // so we only need first parameter of the string slice. + ft := strings.Split(yt, ",")[0] + + if ft == tag { + return p.Field(i), reflect.Value{}, nil + } + } + + // If we can't find the tag but the struct has an 'Other' map field: + if other { + return reflect.Value{}, p.FieldByName("Other"), nil + } + + return reflect.Value{}, reflect.Value{}, fmt.Errorf("unable to find field %q", tag) +} diff --git a/src/go/rpk/pkg/config/params_test.go b/src/go/rpk/pkg/config/params_test.go index e8528bc7883f3..111a1ebaf1898 100644 --- a/src/go/rpk/pkg/config/params_test.go +++ b/src/go/rpk/pkg/config/params_test.go @@ -5,14 +5,13 @@ import ( "testing" "github.com/spf13/afero" - "gopkg.in/yaml.v2" ) func TestParams_Write(t *testing.T) { tests := []struct { name string - inCfg *Config - cfgToWrite *Config + inCfg string + cfgChanges func(*Config) *Config exp string expErr bool }{ @@ -20,51 +19,75 @@ func TestParams_Write(t *testing.T) { name: "create default config file if there is no config file yet", exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: /var/lib/redpanda/data - node_id: 0 - seed_servers: [] - rpc_server: - address: 0.0.0.0 - port: 33145 - kafka_api: - - address: 0.0.0.0 - port: 9092 - admin: - - address: 0.0.0.0 - port: 9644 - developer_mode: true + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - kafka_api: - brokers: - - 0.0.0.0:9092 - admin_api: - addresses: - - 127.0.0.1:9644 - enable_usage_stats: false + kafka_api: + brokers: + - 0.0.0.0:9092 + admin_api: + addresses: + - 127.0.0.1:9644 + enable_usage_stats: false `, }, { - name: "write loaded config", - inCfg: &Config{ConfigFile: "/etc/redpanda/redpanda.yaml", Redpanda: RedpandaConfig{ID: 1}}, - cfgToWrite: &Config{ConfigFile: "/etc/redpanda/redpanda.yaml", Redpanda: RedpandaConfig{ID: 6}}, + name: "write loaded config", + inCfg: `config_file: /etc/redpanda/redpanda.yaml +redpanda: + data_directory: "" + node_id: 1 + rack: my_rack +`, + cfgChanges: func(c *Config) *Config { + c.Redpanda.ID = 6 + return c + }, exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: "" - node_id: 6 + data_directory: "" + node_id: 6 + rack: my_rack +`, + }, + { + name: "write empty structs", + inCfg: `config_file: /etc/redpanda/redpanda.yaml +rpk: + tls: + truststore_file: "" + cert_file: "" + key_file: "" +`, + cfgChanges: func(c *Config) *Config { + c.Rpk.KafkaAPI.Brokers = []string{"127.0.1.1:9647"} + return c + }, + exp: `rpk: + tls: {} + kafka_api: + brokers: + - 127.0.1.1:9647 `, }, } - for _, test := range tests { t.Run(test.name, func(t *testing.T) { fs := afero.NewMemMapFs() - if test.inCfg != nil { - b, err := yaml.Marshal(test.inCfg) - if err != nil { - t.Errorf("marshal error of initial config file: %s", err) - return - } - err = afero.WriteFile(fs, test.inCfg.ConfigFile, b, 0o644) + if test.inCfg != "" { + // We assume for this test that all files will be in the default location. + err := afero.WriteFile(fs, "/etc/redpanda/redpanda.yaml", []byte(test.inCfg), 0o644) if err != nil { t.Errorf("unexpected error while writing initial config file: %s", err) return @@ -83,8 +106,8 @@ redpanda: path = Default().ConfigFile } - if test.cfgToWrite != nil { - cfg = test.cfgToWrite + if test.cfgChanges != nil { + cfg = test.cfgChanges(cfg) } err = cfg.Write(fs) diff --git a/src/go/rpk/pkg/config/schema.go b/src/go/rpk/pkg/config/schema.go index fb5f9aa629152..c3440460a4f1c 100644 --- a/src/go/rpk/pkg/config/schema.go +++ b/src/go/rpk/pkg/config/schema.go @@ -21,19 +21,19 @@ type Config struct { file *Config loadedPath string - NodeUUID string `yaml:"node_uuid,omitempty" mapstructure:"node_uuid,omitempty" json:"nodeUuid"` - Organization string `yaml:"organization,omitempty" mapstructure:"organization,omitempty" json:"organization"` - LicenseKey string `yaml:"license_key,omitempty" mapstructure:"license_key,omitempty" json:"licenseKey"` - ClusterID string `yaml:"cluster_id,omitempty" mapstructure:"cluster_id,omitempty" json:"clusterId"` - ConfigFile string `yaml:"config_file" mapstructure:"config_file" json:"configFile"` - Redpanda RedpandaConfig `yaml:"redpanda" mapstructure:"redpanda" json:"redpanda"` - Rpk RpkConfig `yaml:"rpk" mapstructure:"rpk" json:"rpk"` - Pandaproxy *Pandaproxy `yaml:"pandaproxy,omitempty" mapstructure:"pandaproxy,omitempty" json:"pandaproxy,omitempty"` - PandaproxyClient *KafkaClient `yaml:"pandaproxy_client,omitempty" mapstructure:"pandaproxy_client,omitempty" json:"pandaproxyClient,omitempty"` - SchemaRegistry *SchemaRegistry `yaml:"schema_registry,omitempty" mapstructure:"schema_registry,omitempty" json:"schemaRegistry,omitempty"` - SchemaRegistryClient *KafkaClient `yaml:"schema_registry_client,omitempty" mapstructure:"schema_registry_client,omitempty" json:"schemaRegistryClient,omitempty"` + NodeUUID string `yaml:"node_uuid,omitempty" json:"node_uuid"` + Organization string `yaml:"organization,omitempty" json:"organization"` + LicenseKey string `yaml:"license_key,omitempty" json:"license_key"` + ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id"` + ConfigFile string `yaml:"config_file" json:"config_file"` + Redpanda RedpandaConfig `yaml:"redpanda" json:"redpanda"` + Rpk RpkConfig `yaml:"rpk" json:"rpk"` + Pandaproxy *Pandaproxy `yaml:"pandaproxy,omitempty" json:"pandaproxy,omitempty"` + PandaproxyClient *KafkaClient `yaml:"pandaproxy_client,omitempty" json:"pandaproxy_client,omitempty"` + SchemaRegistry *SchemaRegistry `yaml:"schema_registry,omitempty" json:"schema_registry,omitempty"` + SchemaRegistryClient *KafkaClient `yaml:"schema_registry_client,omitempty" json:"schema_registry_client,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Other map[string]interface{} `yaml:",inline"` } // File returns the configuration as read from a file, with no defaults @@ -44,67 +44,67 @@ func (c *Config) File() *Config { } type RedpandaConfig struct { - Directory string `yaml:"data_directory" mapstructure:"data_directory" json:"dataDirectory"` - ID int `yaml:"node_id" mapstructure:"node_id" json:"id"` - Rack string `yaml:"rack,omitempty" mapstructure:"rack" json:"rack"` - SeedServers []SeedServer `yaml:"seed_servers" mapstructure:"seed_servers" json:"seedServers"` - RPCServer SocketAddress `yaml:"rpc_server" mapstructure:"rpc_server" json:"rpcServer"` - RPCServerTLS []ServerTLS `yaml:"rpc_server_tls,omitempty" mapstructure:"rpc_server_tls,omitempty" json:"rpcServerTls"` - KafkaAPI []NamedSocketAddress `yaml:"kafka_api" mapstructure:"kafka_api" json:"kafkaApi"` - KafkaAPITLS []ServerTLS `yaml:"kafka_api_tls,omitempty" mapstructure:"kafka_api_tls,omitempty" json:"kafkaApiTls"` - AdminAPI []NamedSocketAddress `yaml:"admin" mapstructure:"admin" json:"admin"` - AdminAPITLS []ServerTLS `yaml:"admin_api_tls,omitempty" mapstructure:"admin_api_tls,omitempty" json:"adminApiTls"` - CoprocSupervisorServer SocketAddress `yaml:"coproc_supervisor_server,omitempty" mapstructure:"coproc_supervisor_server" json:"coprocSupervisorServer"` - AdminAPIDocDir string `yaml:"admin_api_doc_dir,omitempty" mapstructure:"admin_api_doc_dir" json:"adminApiDocDir"` - DashboardDir string `yaml:"dashboard_dir,omitempty" mapstructure:"dashboard_dir" json:"dashboardDir"` - CloudStorageCacheDirectory string `yaml:"cloud_storage_cache_directory,omitempty" mapstructure:"cloud_storage_cache_directory" json:"CloudStorageCacheDirectory"` - AdvertisedRPCAPI *SocketAddress `yaml:"advertised_rpc_api,omitempty" mapstructure:"advertised_rpc_api,omitempty" json:"advertisedRpcApi,omitempty"` - AdvertisedKafkaAPI []NamedSocketAddress `yaml:"advertised_kafka_api,omitempty" mapstructure:"advertised_kafka_api,omitempty" json:"advertisedKafkaApi,omitempty"` - DeveloperMode bool `yaml:"developer_mode" mapstructure:"developer_mode" json:"developerMode"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Directory string `yaml:"data_directory" json:"data_directory"` + ID int `yaml:"node_id" json:"node_id"` + Rack string `yaml:"rack,omitempty" json:"rack"` + SeedServers []SeedServer `yaml:"seed_servers" json:"seed_servers"` + RPCServer SocketAddress `yaml:"rpc_server" json:"rpc_server"` + RPCServerTLS []ServerTLS `yaml:"rpc_server_tls,omitempty" json:"rpc_server_tls"` + KafkaAPI []NamedSocketAddress `yaml:"kafka_api" json:"kafka_api"` + KafkaAPITLS []ServerTLS `yaml:"kafka_api_tls,omitempty" json:"kafka_api_tls"` + AdminAPI []NamedSocketAddress `yaml:"admin" json:"admin"` + AdminAPITLS []ServerTLS `yaml:"admin_api_tls,omitempty" json:"admin_api_tls"` + CoprocSupervisorServer SocketAddress `yaml:"coproc_supervisor_server,omitempty" json:"coproc_supervisor_server"` + AdminAPIDocDir string `yaml:"admin_api_doc_dir,omitempty" json:"admin_api_doc_dir"` + DashboardDir string `yaml:"dashboard_dir,omitempty" json:"dashboard_dir"` + CloudStorageCacheDirectory string `yaml:"cloud_storage_cache_directory,omitempty" json:"cloud_storage_cache_directory"` + AdvertisedRPCAPI *SocketAddress `yaml:"advertised_rpc_api,omitempty" json:"advertised_rpc_api,omitempty"` + AdvertisedKafkaAPI []NamedSocketAddress `yaml:"advertised_kafka_api,omitempty" json:"advertised_kafka_api,omitempty"` + DeveloperMode bool `yaml:"developer_mode" json:"developer_mode"` + Other map[string]interface{} `yaml:",inline"` } type Pandaproxy struct { - PandaproxyAPI []NamedSocketAddress `yaml:"pandaproxy_api,omitempty" mapstructure:"pandaproxy_api,omitempty" json:"pandaproxyApi,omitempty"` - PandaproxyAPITLS []ServerTLS `yaml:"pandaproxy_api_tls,omitempty" mapstructure:"pandaproxy_api_tls,omitempty" json:"pandaproxyApiTls,omitempty"` - AdvertisedPandaproxyAPI []NamedSocketAddress `yaml:"advertised_pandaproxy_api,omitempty" mapstructure:"advertised_pandaproxy_api,omitempty" json:"advertisedPandaproxyApi,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + PandaproxyAPI []NamedSocketAddress `yaml:"pandaproxy_api,omitempty" json:"pandaproxy_api,omitempty"` + PandaproxyAPITLS []ServerTLS `yaml:"pandaproxy_api_tls,omitempty" json:"pandaproxy_api_tls,omitempty"` + AdvertisedPandaproxyAPI []NamedSocketAddress `yaml:"advertised_pandaproxy_api,omitempty" json:"advertised_pandaproxy_api,omitempty"` + Other map[string]interface{} `yaml:",inline"` } type SchemaRegistry struct { - SchemaRegistryAPI []NamedSocketAddress `yaml:"schema_registry_api,omitempty" mapstructure:"schema_registry_api,omitempty" json:"schemaRegistryApi,omitempty"` - SchemaRegistryAPITLS []ServerTLS `yaml:"schema_registry_api_tls,omitempty" mapstructure:"schema_registry_api_tls,omitempty" json:"schemaRegistryApiTls,omitempty"` - SchemaRegistryReplicationFactor *int `yaml:"schema_registry_replication_factor,omitempty" mapstructure:"schema_registry_replication_factor,omitempty" json:"schemaRegistryReplicationFactor,omitempty"` + SchemaRegistryAPI []NamedSocketAddress `yaml:"schema_registry_api,omitempty" json:"schema_registry_api,omitempty"` + SchemaRegistryAPITLS []ServerTLS `yaml:"schema_registry_api_tls,omitempty" json:"schema_registry_api_tls,omitempty"` + SchemaRegistryReplicationFactor *int `yaml:"schema_registry_replication_factor,omitempty" json:"schema_registry_replication_factor,omitempty"` } type KafkaClient struct { - Brokers []SocketAddress `yaml:"brokers,omitempty" mapstructure:"brokers,omitempty" json:"brokers,omitempty"` - BrokerTLS ServerTLS `yaml:"broker_tls,omitempty" mapstructure:"broker_tls,omitempty" json:"brokerTls,omitempty"` - SASLMechanism *string `yaml:"sasl_mechanism,omitempty" mapstructure:"sasl_mechanism,omitempty" json:"saslMechanism,omitempty"` - SCRAMUsername *string `yaml:"scram_username,omitempty" mapstructure:"scram_username,omitempty" json:"scramUsername,omitempty"` - SCRAMPassword *string `yaml:"scram_password,omitempty" mapstructure:"scram_password,omitempty" json:"scramPassword,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Brokers []SocketAddress `yaml:"brokers,omitempty" json:"brokers,omitempty"` + BrokerTLS ServerTLS `yaml:"broker_tls,omitempty" json:"broker_tls,omitempty"` + SASLMechanism *string `yaml:"sasl_mechanism,omitempty" json:"sasl_mechanism,omitempty"` + SCRAMUsername *string `yaml:"scram_username,omitempty" json:"scram_username,omitempty"` + SCRAMPassword *string `yaml:"scram_password,omitempty" json:"scram_password,omitempty"` + Other map[string]interface{} `yaml:",inline"` } type SeedServer struct { - Host SocketAddress `yaml:"host" mapstructure:"host" json:"host"` + Host SocketAddress `yaml:"host" json:"host"` } type SocketAddress struct { - Address string `yaml:"address" mapstructure:"address" json:"address"` - Port int `yaml:"port" mapstructure:"port" json:"port"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` } type NamedSocketAddress struct { - Address string `yaml:"address" mapstructure:"address" json:"address"` - Port int `yaml:"port" mapstructure:"port" json:"port"` - Name string `yaml:"name,omitempty" mapstructure:"name,omitempty" json:"name,omitempty"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` } type TLS struct { - KeyFile string `yaml:"key_file,omitempty" mapstructure:"key_file,omitempty" json:"keyFile"` - CertFile string `yaml:"cert_file,omitempty" mapstructure:"cert_file,omitempty" json:"certFile"` - TruststoreFile string `yaml:"truststore_file,omitempty" mapstructure:"truststore_file,omitempty" json:"truststoreFile"` + KeyFile string `yaml:"key_file,omitempty" json:"key_file"` + CertFile string `yaml:"cert_file,omitempty" json:"cert_file"` + TruststoreFile string `yaml:"truststore_file,omitempty" json:"truststore_file"` } func (t *TLS) Config(fs afero.Fs) (*tls.Config, error) { @@ -129,62 +129,62 @@ func (t *TLS) Config(fs afero.Fs) (*tls.Config, error) { } type ServerTLS struct { - Name string `yaml:"name,omitempty" mapstructure:"name,omitempty" json:"name"` - KeyFile string `yaml:"key_file,omitempty" mapstructure:"key_file,omitempty" json:"keyFile"` - CertFile string `yaml:"cert_file,omitempty" mapstructure:"cert_file,omitempty" json:"certFile"` - TruststoreFile string `yaml:"truststore_file,omitempty" mapstructure:"truststore_file,omitempty" json:"truststoreFile"` - Enabled bool `yaml:"enabled,omitempty" mapstructure:"enabled,omitempty" json:"enabled"` - RequireClientAuth bool `yaml:"require_client_auth,omitempty" mapstructure:"require_client_auth,omitempty" json:"requireClientAuth"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Name string `yaml:"name,omitempty" json:"name"` + KeyFile string `yaml:"key_file,omitempty" json:"key_file"` + CertFile string `yaml:"cert_file,omitempty" json:"cert_file"` + TruststoreFile string `yaml:"truststore_file,omitempty" json:"truststore_file"` + Enabled bool `yaml:"enabled,omitempty" json:"enabled"` + RequireClientAuth bool `yaml:"require_client_auth,omitempty" json:"require_client_auth"` + Other map[string]interface{} `yaml:",inline" ` } type RpkConfig struct { // Deprecated 2021-07-1 - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` // Deprecated 2021-07-1 - SASL *SASL `yaml:"sasl,omitempty" mapstructure:"sasl,omitempty" json:"sasl,omitempty"` - - KafkaAPI RpkKafkaAPI `yaml:"kafka_api,omitempty" mapstructure:"kafka_api,omitempty" json:"kafkaApi"` - AdminAPI RpkAdminAPI `yaml:"admin_api,omitempty" mapstructure:"admin_api,omitempty" json:"adminApi"` - AdditionalStartFlags []string `yaml:"additional_start_flags,omitempty" mapstructure:"additional_start_flags,omitempty" json:"additionalStartFlags"` - EnableUsageStats bool `yaml:"enable_usage_stats" mapstructure:"enable_usage_stats" json:"enableUsageStats"` - TuneNetwork bool `yaml:"tune_network" mapstructure:"tune_network" json:"tuneNetwork"` - TuneDiskScheduler bool `yaml:"tune_disk_scheduler" mapstructure:"tune_disk_scheduler" json:"tuneDiskScheduler"` - TuneNomerges bool `yaml:"tune_disk_nomerges" mapstructure:"tune_disk_nomerges" json:"tuneNomerges"` - TuneDiskWriteCache bool `yaml:"tune_disk_write_cache" mapstructure:"tune_disk_write_cache" json:"tuneDiskWriteCache"` - TuneDiskIrq bool `yaml:"tune_disk_irq" mapstructure:"tune_disk_irq" json:"tuneDiskIrq"` - TuneFstrim bool `yaml:"tune_fstrim" mapstructure:"tune_fstrim" json:"tuneFstrim"` - TuneCPU bool `yaml:"tune_cpu" mapstructure:"tune_cpu" json:"tuneCpu"` - TuneAioEvents bool `yaml:"tune_aio_events" mapstructure:"tune_aio_events" json:"tuneAioEvents"` - TuneClocksource bool `yaml:"tune_clocksource" mapstructure:"tune_clocksource" json:"tuneClocksource"` - TuneSwappiness bool `yaml:"tune_swappiness" mapstructure:"tune_swappiness" json:"tuneSwappiness"` - TuneTransparentHugePages bool `yaml:"tune_transparent_hugepages" mapstructure:"tune_transparent_hugepages" json:"tuneTransparentHugePages"` - EnableMemoryLocking bool `yaml:"enable_memory_locking" mapstructure:"enable_memory_locking" json:"enableMemoryLocking"` - TuneCoredump bool `yaml:"tune_coredump" mapstructure:"tune_coredump" json:"tuneCoredump"` - CoredumpDir string `yaml:"coredump_dir,omitempty" mapstructure:"coredump_dir,omitempty" json:"coredumpDir"` - TuneBallastFile bool `yaml:"tune_ballast_file" mapstructure:"tune_ballast_file" json:"tuneBallastFile"` - BallastFilePath string `yaml:"ballast_file_path,omitempty" mapstructure:"ballast_file_path,omitempty" json:"ballastFilePath"` - BallastFileSize string `yaml:"ballast_file_size,omitempty" mapstructure:"ballast_file_size,omitempty" json:"ballastFileSize"` - WellKnownIo string `yaml:"well_known_io,omitempty" mapstructure:"well_known_io,omitempty" json:"wellKnownIo"` - Overprovisioned bool `yaml:"overprovisioned" mapstructure:"overprovisioned" json:"overprovisioned"` - SMP *int `yaml:"smp,omitempty" mapstructure:"smp,omitempty" json:"smp,omitempty"` + SASL *SASL `yaml:"sasl,omitempty" json:"sasl,omitempty"` + + KafkaAPI RpkKafkaAPI `yaml:"kafka_api,omitempty" json:"kafka_api"` + AdminAPI RpkAdminAPI `yaml:"admin_api,omitempty" json:"admin_api"` + AdditionalStartFlags []string `yaml:"additional_start_flags,omitempty" json:"additional_start_flags"` + EnableUsageStats bool `yaml:"enable_usage_stats" json:"enable_usage_stats"` + TuneNetwork bool `yaml:"tune_network" json:"tune_network"` + TuneDiskScheduler bool `yaml:"tune_disk_scheduler" json:"tune_disk_scheduler"` + TuneNomerges bool `yaml:"tune_disk_nomerges" json:"tune_disk_nomerges"` + TuneDiskWriteCache bool `yaml:"tune_disk_write_cache" json:"tune_disk_write_cache"` + TuneDiskIrq bool `yaml:"tune_disk_irq" json:"tune_disk_irq"` + TuneFstrim bool `yaml:"tune_fstrim" json:"tune_fstrim"` + TuneCPU bool `yaml:"tune_cpu" json:"tune_cpu"` + TuneAioEvents bool `yaml:"tune_aio_events" json:"tune_aio_events"` + TuneClocksource bool `yaml:"tune_clocksource" json:"tune_clocksource"` + TuneSwappiness bool `yaml:"tune_swappiness" json:"tune_swappiness"` + TuneTransparentHugePages bool `yaml:"tune_transparent_hugepages" json:"tune_transparent_hugepages"` + EnableMemoryLocking bool `yaml:"enable_memory_locking" json:"enable_memory_locking"` + TuneCoredump bool `yaml:"tune_coredump" json:"tune_coredump"` + CoredumpDir string `yaml:"coredump_dir,omitempty" json:"coredump_dir"` + TuneBallastFile bool `yaml:"tune_ballast_file" json:"tune_ballast_file"` + BallastFilePath string `yaml:"ballast_file_path,omitempty" json:"ballast_file_path"` + BallastFileSize string `yaml:"ballast_file_size,omitempty" json:"ballast_file_size"` + WellKnownIo string `yaml:"well_known_io,omitempty" json:"well_known_io"` + Overprovisioned bool `yaml:"overprovisioned" json:"overprovisioned"` + SMP *int `yaml:"smp,omitempty" json:"smp,omitempty"` } type RpkKafkaAPI struct { - Brokers []string `yaml:"brokers,omitempty" mapstructure:"brokers,omitempty" json:"brokers"` - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` - SASL *SASL `yaml:"sasl,omitempty" mapstructure:"sasl,omitempty" json:"sasl,omitempty"` + Brokers []string `yaml:"brokers,omitempty" json:"brokers"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` + SASL *SASL `yaml:"sasl,omitempty" json:"sasl,omitempty"` } type RpkAdminAPI struct { - Addresses []string `yaml:"addresses,omitempty" mapstructure:"addresses,omitempty" json:"addresses"` - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` + Addresses []string `yaml:"addresses,omitempty" json:"addresses"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` } type SASL struct { - User string `yaml:"user,omitempty" mapstructure:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" mapstructure:"password,omitempty" json:"password,omitempty"` - Mechanism string `yaml:"type,omitempty" mapstructure:"type,omitempty" json:"type,omitempty"` + User string `yaml:"user,omitempty" json:"user,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` + Mechanism string `yaml:"type,omitempty" json:"type,omitempty"` } func (c *Config) PIDFile() string { diff --git a/src/go/rpk/pkg/config/weak.go b/src/go/rpk/pkg/config/weak.go index 96e97d45b9ad6..0c9b6a2c4cca5 100644 --- a/src/go/rpk/pkg/config/weak.go +++ b/src/go/rpk/pkg/config/weak.go @@ -10,8 +10,9 @@ package config import ( + "errors" "fmt" - "os" + "reflect" "strconv" "gopkg.in/yaml.v3" @@ -22,6 +23,7 @@ import ( // // The use of this file is to support our transition to a strongly typed // config file and our migration away from viper and mapstructure. +// TODO: Print deprecation warning when using weak types https://github.com/redpanda-data/redpanda/issues/5262 // weakBool is an intermediary boolean type to be used during our transition // to strictly typed configuration parameters. This will allow us to support @@ -46,18 +48,15 @@ func (wb *weakBool) UnmarshalYAML(n *yaml.Node) error { if err != nil { return fmt.Errorf("cannot parse '%s' as bool: %s", n.Value, err) } - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from int to bool is deprecated and will be removed in the future") *wb = ni != 0 return nil case "!!str": // it accepts 1, t, T, TRUE, true, True, 0, f, F nb, err := strconv.ParseBool(n.Value) if err == nil { - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from str to bool is deprecated and will be removed in the future") *wb = weakBool(nb) return nil } else if n.Value == "" { - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from str to bool is deprecated and will be removed in the future") *wb = false return nil } else { @@ -94,7 +93,6 @@ func (wi *weakInt) UnmarshalYAML(n *yaml.Node) error { if err != nil { return fmt.Errorf("cannot parse '%s' as an integer: %s", str, err) } - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from str to int is deprecated and will be removed in the future") *wi = weakInt(ni) return nil case "!!bool": @@ -102,7 +100,6 @@ func (wi *weakInt) UnmarshalYAML(n *yaml.Node) error { if err != nil { return fmt.Errorf("cannot parse '%s' as an integer: %s", n.Value, err) } - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from bool to int is deprecated and will be removed in the future") if nb { *wi = 1 return nil @@ -132,7 +129,6 @@ func (ws *weakString) UnmarshalYAML(n *yaml.Node) error { if err != nil { return fmt.Errorf("cannot parse '%s' as a boolean: %s", n.Value, err) } - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from bool to str is deprecated and will be removed in the future") if nb { *ws = "1" return nil @@ -140,7 +136,6 @@ func (ws *weakString) UnmarshalYAML(n *yaml.Node) error { *ws = "0" return nil case "!!int", "!!float": - fmt.Fprintln(os.Stderr, "redpanda.yaml: conversion from numbers to str is deprecated and will be removed in the future") *ws = weakString(n.Value) return nil default: @@ -172,7 +167,6 @@ func (wsa *weakStringArray) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %q must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", n.Value) *wsa = []string{string(single)} return nil } @@ -197,7 +191,6 @@ func (s *socketAddresses) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) *s = []SocketAddress{single} return nil } @@ -222,7 +215,6 @@ func (nsa *namedSocketAddresses) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) *nsa = []NamedSocketAddress{single} return nil } @@ -246,7 +238,7 @@ func (s *serverTLSArray) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) + // do not log serverTLS because the Other field may contain a secret *s = []ServerTLS{single} return nil } @@ -270,7 +262,6 @@ func (ss *seedServers) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) *ss = []SeedServer{single} return nil } @@ -333,6 +324,7 @@ func (rpc *RedpandaConfig) UnmarshalYAML(n *yaml.Node) error { DeveloperMode weakBool `yaml:"developer_mode"` Other map[string]interface{} `yaml:",inline"` } + if err := n.Decode(&internal); err != nil { return err } @@ -392,12 +384,6 @@ func (rpkc *RpkConfig) UnmarshalYAML(n *yaml.Node) error { if err := n.Decode(&internal); err != nil { return err } - if internal.TLS != nil { - fmt.Fprintln(os.Stderr, "redpanda.yaml: rpk.tls has been replaced with rpk.kafka_api.tls and will be removed in the future") - } - if internal.SASL != nil { - fmt.Fprintln(os.Stderr, "redpanda.yaml: rpk.sasl has been replaced with rpk.kafka_api.sasl and will be removed in the future") - } rpkc.TLS = internal.TLS rpkc.SASL = internal.SASL @@ -533,6 +519,45 @@ func (s *ServerTLS) UnmarshalYAML(n *yaml.Node) error { return nil } +func (ss *SeedServer) UnmarshalYAML(n *yaml.Node) error { + var internal struct { + // New schema should only contain Address and Port, but we will + // support this under Host also. + Address weakString `yaml:"address"` + Port weakInt `yaml:"port"` + Host SocketAddress `yaml:"host"` + // deprecated + NodeID *weakInt `yaml:"node_id"` + } + if err := n.Decode(&internal); err != nil { + return err + } + if internal.NodeID != nil { + fmt.Println("redpanda yaml: redpanda.seed_server.node_id is deprecated and unused") + } + + if internal.Address != "" || internal.Port != 0 { + embedded := SocketAddress{string(internal.Address), int(internal.Port)} + nested := internal.Host + + embeddedZero := reflect.DeepEqual(embedded, SocketAddress{}) + nestedZero := reflect.DeepEqual(nested, SocketAddress{}) + + if !embeddedZero && !nestedZero && !reflect.DeepEqual(embedded, nested) { + return errors.New("redpanda.yaml redpanda.seed_server: nested host differs from address and port fields; only one must be set") + } + + ss.Host = embedded + if embeddedZero { + ss.Host = nested + } + return nil + } + + ss.Host = internal.Host + return nil +} + func (sa *SocketAddress) UnmarshalYAML(n *yaml.Node) error { var internal struct { Address weakString `yaml:"address"` diff --git a/src/go/rpk/pkg/config/weak_test.go b/src/go/rpk/pkg/config/weak_test.go index 3f578868a8635..00f4b3cb4f273 100644 --- a/src/go/rpk/pkg/config/weak_test.go +++ b/src/go/rpk/pkg/config/weak_test.go @@ -545,7 +545,7 @@ func TestSeedServers(t *testing.T) { port: 80 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, + {SocketAddress{"0.0.0.1", 80}}, }, }, { @@ -559,8 +559,8 @@ func TestSeedServers(t *testing.T) { port: 90 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, - {Host: SocketAddress{"0.0.0.2", 90}}, + {SocketAddress{"0.0.0.1", 80}}, + {SocketAddress{"0.0.0.2", 90}}, }, }, { @@ -593,6 +593,142 @@ func TestSeedServers(t *testing.T) { } } +func TestSeedServer(t *testing.T) { + for _, test := range []struct { + name string + data string + exp SeedServer + expErr bool + }{ + { + name: "with node_id", + data: `test_server: + node_id: 1 + host: + address: 192.168.10.1 + port: 33145 +`, + exp: SeedServer{ + SocketAddress{"192.168.10.1", 33145}, + }, + }, + { + name: "with host", + data: `test_server: + host: + address: "0.0.0.1" + port: 80 +`, + exp: SeedServer{ + SocketAddress{"0.0.0.1", 80}, + }, + }, + { + name: "address and port", + data: `test_server: + address: "1.0.0.1" + port: 80 +`, + exp: SeedServer{ + SocketAddress{"1.0.0.1", 80}, + }, + }, + { + name: "equal host and address & port", + data: `test_server: + host: + address: "0.0.0.1" + port: 80 + address: "0.0.0.1" + port: 80 +`, + exp: SeedServer{ + SocketAddress{"0.0.0.1", 80}, + }, + }, + { + name: "host.address and port", + data: `test_server: + host: + address: "0.0.0.1" + port: 80 +`, + expErr: true, + }, + { + name: "address and host.port", + data: `test_server: + host: + port: 80 + address: "0.0.0.1" +`, + expErr: true, + }, + { + name: "address different from host.address", + data: `test_server: + host: + address: "0.0.0.1" + port: 80 + address: "0.2.0.1" + port: 80 +`, + expErr: true, + }, + { + name: "port different from host.port", + data: `test_server: + host: + address: "0.0.0.1" + port: 82 + address: "0.0.0.1" + port: 80 +`, + expErr: true, + }, + { + name: "equal host.address and address", + data: `test_server: + host: + address: "0.0.0.1" + address: "0.0.0.1" +`, + exp: SeedServer{ + SocketAddress{Address: "0.0.0.1"}, + }, + }, + { + name: "equal host.port and port", + data: `test_server: + host: + port: 82 + port: 82 +`, + exp: SeedServer{ + SocketAddress{Port: 82}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + var ts struct { + Ss SeedServer `yaml:"test_server"` + } + err := yaml.Unmarshal([]byte(test.data), &ts) + + gotErr := err != nil + if gotErr != test.expErr { + t.Errorf("input %q: got err? %v, exp err? %v; error: %v", + test.data, gotErr, test.expErr, err) + return + } + if test.expErr { + return + } + require.Equal(t, test.exp, ts.Ss) + }) + } +} + func TestConfig_UnmarshalYAML(t *testing.T) { for _, test := range []struct { name string @@ -647,9 +783,8 @@ redpanda: name: external port: 9093 seed_servers: - - host: - address: 192.168.0.1 - port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -761,7 +896,7 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {SocketAddress{"192.168.0.1", 33145}}, }, Other: map[string]interface{}{ "enable_admin_api": true, @@ -876,9 +1011,15 @@ redpanda: name: external port: 9093 seed_servers: + - host: + address: 192.168.0.1 + port: 33145 + - node_id: "0" host: address: 192.168.0.1 port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -989,7 +1130,9 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {SocketAddress{"192.168.0.1", 33145}}, + {SocketAddress{"192.168.0.1", 33145}}, + {SocketAddress{"192.168.0.1", 33145}}, }, Other: map[string]interface{}{ "enable_admin_api": true, diff --git a/src/go/rpk/pkg/plugin/manifest.go b/src/go/rpk/pkg/plugin/manifest.go index e631ccbb07fdb..ecca50478f13b 100644 --- a/src/go/rpk/pkg/plugin/manifest.go +++ b/src/go/rpk/pkg/plugin/manifest.go @@ -15,7 +15,7 @@ import ( "strings" "time" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Manifest represents a plugin manifest, which is essentially a list of diff --git a/src/go/rpk/pkg/tuners/iotune/data.go b/src/go/rpk/pkg/tuners/iotune/data.go index e0af774aa69c1..3c1b631eab18c 100644 --- a/src/go/rpk/pkg/tuners/iotune/data.go +++ b/src/go/rpk/pkg/tuners/iotune/data.go @@ -14,7 +14,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cloud/vendor" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type IoProperties struct { diff --git a/src/go/rpk/pkg/tuners/redpanda_checkers.go b/src/go/rpk/pkg/tuners/redpanda_checkers.go index 68d40ffa300ba..340b5993b9420 100644 --- a/src/go/rpk/pkg/tuners/redpanda_checkers.go +++ b/src/go/rpk/pkg/tuners/redpanda_checkers.go @@ -69,7 +69,7 @@ func NewConfigChecker(conf *config.Config) Checker { Fatal, true, func() (interface{}, error) { - ok, _ := config.Check(conf) + ok, _ := conf.Check() return ok, nil }) } diff --git a/src/go/rpk/pkg/yaml/yaml.go b/src/go/rpk/pkg/yaml/yaml.go deleted file mode 100644 index 47385a64312cb..0000000000000 --- a/src/go/rpk/pkg/yaml/yaml.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package yaml - -import ( - "github.com/spf13/afero" - "gopkg.in/yaml.v2" -) - -func Persist(fs afero.Fs, in interface{}, file string) error { - bytes, err := yaml.Marshal(in) - if err != nil { - return err - } - err = afero.WriteFile(fs, file, bytes, 0o644) - return err -} - -func Read(fs afero.Fs, out interface{}, file string) error { - content, err := afero.ReadFile(fs, file) - if err != nil { - return err - } - err = yaml.Unmarshal(content, out) - if err != nil { - return err - } - return nil -} diff --git a/tests/rptest/tests/rpk_config_test.py b/tests/rptest/tests/rpk_config_test.py index 04ab14608629d..18047df74fa51 100644 --- a/tests/rptest/tests/rpk_config_test.py +++ b/tests/rptest/tests/rpk_config_test.py @@ -39,37 +39,43 @@ def test_config_init(self): # node_uuid: (the uuid is random so we don't compare it) pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false + admin_api: + addresses: + - 127.0.0.1:9644 + coredump_dir: /var/lib/redpanda/coredump + enable_memory_locking: false + enable_usage_stats: false + kafka_api: + brokers: + - 0.0.0.0:9092 + overprovisioned: false + tune_aio_events: false + tune_ballast_file: false + tune_clocksource: false + tune_coredump: false + tune_cpu: false + tune_disk_irq: false + tune_disk_nomerges: false + tune_disk_scheduler: false + tune_disk_write_cache: false + tune_fstrim: false + tune_network: false + tune_swappiness: false + tune_transparent_hugepages: false schema_registry: {} ''' @@ -106,14 +112,19 @@ def test_config_set_single_number(self): with open(os.path.join(d, config_file)) as f: actual_config = yaml.full_load(f.read()) - assert f"{actual_config['redpanda']['admin']['port']}" == value + if f"{actual_config['redpanda']['admin'][0]['port']}" != value: + self.logger.error("Configs differ") + self.logger.error(f"Expected: {value}") + self.logger.error( + f"Actual: {yaml.dump(actual_config['redpanda']['admin'][0]['port'])}" + ) + assert f"{actual_config['redpanda']['admin'][0]['port']}" == value @cluster(num_nodes=3) def test_config_set_yaml(self): n = random.randint(1, len(self.redpanda.nodes)) node = self.redpanda.get_node(n) rpk = RpkRemoteTool(self.redpanda, node) - path = '/etc/redpanda/redpanda.yaml' key = 'redpanda.seed_servers' value = ''' - node_id: 1 @@ -129,14 +140,34 @@ def test_config_set_yaml(self): address: 192.168.10.3 port: 33145 ''' + + expected = ''' +- host: + address: 192.168.10.1 + port: 33145 +- host: + address: 192.168.10.2 + port: 33145 +- host: + address: 192.168.10.3 + port: 33145 +''' + rpk.config_set(key, value, format='yaml') with tempfile.TemporaryDirectory() as d: node.account.copy_from(RedpandaService.NODE_CONFIG_FILE, d) with open(os.path.join(d, 'redpanda.yaml')) as f: - expected_config = yaml.full_load(value) + expected_config = yaml.full_load(expected) actual_config = yaml.full_load(f.read()) + if actual_config['redpanda']['seed_servers'] != expected_config: + self.logger.error("Configs differ") + self.logger.error( + f"Expected: {yaml.dump(expected_config)}") + self.logger.error( + f"Actual: {yaml.dump(actual_config['redpanda']['seed_servers'])}" + ) assert actual_config['redpanda'][ 'seed_servers'] == expected_config @@ -151,19 +182,26 @@ def test_config_set_json(self): rpk.config_set(key, value, format='json') expected_config = yaml.full_load(''' +admin_api: + addresses: + - 127.0.0.1:9644 coredump_dir: /var/lib/redpanda/coredump enable_memory_locking: false -enable_usage_stats: false +enable_usage_stats: false +overprovisioned: false tune_aio_events: true +tune_ballast_file: false tune_clocksource: false tune_coredump: false tune_cpu: true tune_disk_irq: true tune_disk_nomerges: false tune_disk_scheduler: false +tune_disk_write_cache: false tune_fstrim: false tune_network: false tune_swappiness: false +tune_transparent_hugepages: false ''') with tempfile.TemporaryDirectory() as d: @@ -172,6 +210,18 @@ def test_config_set_json(self): with open(os.path.join(d, 'redpanda.yaml')) as f: actual_config = yaml.full_load(f.read()) + assert actual_config['rpk']['kafka_api'] is not None + + # Delete 'kafka_api' so they can be compared since the + # brokers change depending on the container it's running + del actual_config['rpk']['kafka_api'] + + if actual_config['rpk'] != expected_config: + self.logger.error("Configs differ") + self.logger.error( + f"Expected: {yaml.dump(expected_config)}") + self.logger.error( + f"Actual: {yaml.dump(actual_config['rpk'])}") assert actual_config['rpk'] == expected_config @cluster(num_nodes=3, log_allow_list=RESTART_LOG_ALLOW_LIST)