From df1034ebb4725d543517542cd50ecf33420ec9ea Mon Sep 17 00:00:00 2001 From: Rogger Vasquez Date: Tue, 7 Jun 2022 18:26:24 -0500 Subject: [PATCH] rpk: remove viper and mapstructure This commit removes viper and mapstructure These are all the required changes to handle the parsing, weak typing and writes to config file. From now on we will print warning to users so they know what needs to be changed in the config file before we deprecate this weak typing. This also include Yaml V3 upgrade, one notable change is the indent in yaml files. --- src/go/k8s/cmd/configurator/main.go | 4 +- src/go/k8s/go.mod | 11 +- src/go/k8s/go.sum | 18 +- .../configuration/configuration_modes.go | 20 +- src/go/rpk/go.mod | 13 +- src/go/rpk/go.sum | 23 +- .../rpk/pkg/cli/cmd/cluster/config/reset.go | 2 +- src/go/rpk/pkg/cli/cmd/config.go | 5 +- src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go | 2 +- src/go/rpk/pkg/cli/cmd/generate/prometheus.go | 2 +- src/go/rpk/pkg/cli/cmd/mode.go | 6 +- src/go/rpk/pkg/cli/cmd/redpanda.go | 13 +- src/go/rpk/pkg/cli/cmd/redpanda/config.go | 130 +- .../rpk/pkg/cli/cmd/redpanda/config_test.go | 285 +-- src/go/rpk/pkg/cli/cmd/redpanda/mode.go | 28 +- src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go | 71 +- src/go/rpk/pkg/cli/cmd/redpanda/start.go | 92 +- src/go/rpk/pkg/cli/cmd/redpanda/start_test.go | 270 +-- src/go/rpk/pkg/cli/cmd/redpanda/stop.go | 19 +- src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go | 5 +- src/go/rpk/pkg/cli/cmd/root.go | 3 +- src/go/rpk/pkg/cli/cmd/root_darwin.go | 5 +- src/go/rpk/pkg/cli/cmd/root_linux.go | 15 +- src/go/rpk/pkg/cli/cmd/start.go | 7 +- src/go/rpk/pkg/cli/cmd/stop.go | 5 +- src/go/rpk/pkg/config/config.go | 300 +--- src/go/rpk/pkg/config/config_test.go | 1558 +++-------------- src/go/rpk/pkg/config/find.go | 70 - src/go/rpk/pkg/config/find_test.go | 83 - src/go/rpk/pkg/config/license.go | 87 - src/go/rpk/pkg/config/license_test.go | 117 -- src/go/rpk/pkg/config/manager.go | 425 ----- src/go/rpk/pkg/config/params.go | 147 +- src/go/rpk/pkg/config/params_test.go | 48 +- src/go/rpk/pkg/config/schema.go | 3 +- src/go/rpk/pkg/config/weak.go | 27 +- src/go/rpk/pkg/config/weak_test.go | 23 +- src/go/rpk/pkg/plugin/manifest.go | 2 +- src/go/rpk/pkg/tuners/iotune/data.go | 2 +- src/go/rpk/pkg/yaml/yaml.go | 36 - tests/rptest/tests/rpk_config_test.py | 86 +- 41 files changed, 893 insertions(+), 3175 deletions(-) delete mode 100644 src/go/rpk/pkg/config/find.go delete mode 100644 src/go/rpk/pkg/config/find_test.go delete mode 100644 src/go/rpk/pkg/config/license.go delete mode 100644 src/go/rpk/pkg/config/license_test.go delete mode 100644 src/go/rpk/pkg/config/manager.go delete mode 100644 src/go/rpk/pkg/yaml/yaml.go diff --git a/src/go/k8s/cmd/configurator/main.go b/src/go/k8s/cmd/configurator/main.go index b019540579ec4..d83e003dc520b 100644 --- a/src/go/k8s/cmd/configurator/main.go +++ b/src/go/k8s/cmd/configurator/main.go @@ -99,8 +99,8 @@ func main() { log.Print(c.String()) fs := afero.NewOsFs() - mgr := config.NewManager(fs) - cfg, err := mgr.Read(path.Join(c.configSourceDir, "redpanda.yaml")) + p := config.Params{ConfigPath: path.Join(c.configSourceDir, "redpanda.yaml")} + cfg, err := p.Load(fs) if err != nil { log.Fatalf("%s", fmt.Errorf("unable to read the redpanda configuration file: %w", err)) } diff --git a/src/go/k8s/go.mod b/src/go/k8s/go.mod index 84024a5ce8978..e1724e93623c8 100644 --- a/src/go/k8s/go.mod +++ b/src/go/k8s/go.mod @@ -15,7 +15,7 @@ require ( github.com/redpanda-data/redpanda/src/go/rpk v0.0.0-00010101000000-000000000000 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.21.4 k8s.io/apimachinery v0.21.4 k8s.io/client-go v0.21.4 @@ -48,16 +48,12 @@ require ( github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -65,12 +61,8 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/sethgrid/pester v1.1.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect - github.com/spf13/cast v1.3.0 // indirect github.com/spf13/cobra v1.1.3 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/twmb/tlscfg v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -86,7 +78,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.52.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.21.4 // indirect diff --git a/src/go/k8s/go.sum b/src/go/k8s/go.sum index 5701a2bb3447f..19114ba780fbf 100644 --- a/src/go/k8s/go.sum +++ b/src/go/k8s/go.sum @@ -58,7 +58,6 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= @@ -324,7 +323,6 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -364,7 +362,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -400,7 +397,6 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -430,7 +426,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -467,8 +462,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= @@ -516,7 +509,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -577,9 +569,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -587,7 +577,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -595,7 +584,6 @@ github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -604,7 +592,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -618,7 +605,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -1001,7 +987,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1023,8 +1008,9 @@ gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/k8s/pkg/resources/configuration/configuration_modes.go b/src/go/k8s/pkg/resources/configuration/configuration_modes.go index 30d46e2898a55..22ac7a42fd456 100644 --- a/src/go/k8s/pkg/resources/configuration/configuration_modes.go +++ b/src/go/k8s/pkg/resources/configuration/configuration_modes.go @@ -14,9 +14,6 @@ import ( "reflect" "strconv" "strings" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/spf13/afero" ) // GlobalConfigurationMode changes the behavior of the global configuration when reading/writing properties. @@ -65,33 +62,20 @@ func (r globalConfigurationModeClassic) GetAdditionalRedpandaProperty( func (r globalConfigurationModeClassic) SetAdditionalFlatProperties( targetConfig *GlobalConfiguration, props map[string]string, ) error { - // all properties are node properties in the classic setting - mgr := config.NewManager(afero.NewOsFs()) - err := mgr.Merge(&targetConfig.NodeConfiguration) - if err != nil { - return fmt.Errorf("merging node configuration: %w", err) - } - // Add arbitrary parameters to configuration for k, v := range props { if builtInType(v) { - err = mgr.Set(k, v, "single") + err := targetConfig.NodeConfiguration.Set(k, v, "single") if err != nil { return fmt.Errorf("setting built-in type: %w", err) } } else { - err = mgr.Set(k, v, "") + err := targetConfig.NodeConfiguration.Set(k, v, "") if err != nil { return fmt.Errorf("setting complex type: %w", err) } } } - - newRpCfg, err := mgr.Get() - if err != nil { - return fmt.Errorf("getting redpanda node configuration: %w", err) - } - targetConfig.NodeConfiguration = *newRpCfg return nil } diff --git a/src/go/rpk/go.mod b/src/go/rpk/go.mod index 454c82a4709a7..9fcaaf55d4daf 100644 --- a/src/go/rpk/go.mod +++ b/src/go/rpk/go.mod @@ -17,7 +17,6 @@ require ( github.com/google/uuid v1.1.1 github.com/hashicorp/go-multierror v1.1.0 github.com/lorenzosaino/go-sysctl v0.1.0 - github.com/mitchellh/mapstructure v1.4.1 github.com/olekukonko/tablewriter v0.0.1 github.com/opencontainers/image-spec v1.0.1 github.com/pkg/errors v0.9.1 @@ -29,7 +28,6 @@ require ( github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 github.com/tklauser/go-sysconf v0.1.0 github.com/twmb/franz-go v1.6.0 @@ -40,8 +38,7 @@ require ( golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -49,20 +46,17 @@ require ( github.com/containerd/containerd v1.4.8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/godbus/dbus/v5 v5.0.3 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/protobuf v1.4.2 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect @@ -71,12 +65,8 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tklauser/numcpus v0.1.0 // indirect golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect @@ -86,6 +76,5 @@ require ( google.golang.org/grpc v1.27.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.51.0 // indirect gotest.tools/v3 v3.0.3 // indirect ) diff --git a/src/go/rpk/go.sum b/src/go/rpk/go.sum index f96d58a2c356a..b83b8a7b83d4a 100644 --- a/src/go/rpk/go.sum +++ b/src/go/rpk/go.sum @@ -16,7 +16,6 @@ github.com/AlecAivazis/survey/v2 v2.3.2 h1:TqTB+aDDCLYhf9/bD2TwSO8u8jDSmMUd2SUVO github.com/AlecAivazis/survey/v2 v2.3.2/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= @@ -77,8 +76,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -123,7 +120,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -149,7 +145,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -164,7 +159,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -190,7 +184,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lorenzosaino/go-sysctl v0.1.0 h1:BfWlLYErjQeCb0TB3kzIq5nsVVjKqyT0NKvWqifz7gE= github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -216,8 +209,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -233,7 +224,6 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -275,9 +265,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= @@ -285,16 +273,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -304,7 +289,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tklauser/go-sysconf v0.1.0 h1:f4fOH/B3HZYy2wz7o9Vr5i4dBhOKcedooHq7/4c0hxQ= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -401,7 +385,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -490,18 +473,16 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go index e2bc13650ceb9..53d5b2e833196 100644 --- a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go +++ b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go @@ -16,7 +16,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func newForceResetCommand(fs afero.Fs) *cobra.Command { diff --git a/src/go/rpk/pkg/cli/cmd/config.go b/src/go/rpk/pkg/cli/cmd/config.go index 6061658cce7e1..1051630ed50b2 100644 --- a/src/go/rpk/pkg/cli/cmd/config.go +++ b/src/go/rpk/pkg/cli/cmd/config.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewConfigCommand(fs, mgr), + redpanda.NewConfigCommand(fs), "rpk redpanda config", ) } diff --git a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go index f4c900d636bd9..4653cd321b272 100644 --- a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go +++ b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go @@ -43,7 +43,7 @@ import ( "github.com/spf13/afero" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func executeBundle( diff --git a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go index 5d9a32f2c48e0..8d78a55234039 100644 --- a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go +++ b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/cobra" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type ScrapeConfig struct { diff --git a/src/go/rpk/pkg/cli/cmd/mode.go b/src/go/rpk/pkg/cli/cmd/mode.go index a300c74111a24..802d8b1b1b242 100644 --- a/src/go/rpk/pkg/cli/cmd/mode.go +++ b/src/go/rpk/pkg/cli/cmd/mode.go @@ -15,13 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewModeCommand(mgr), + redpanda.NewModeCommand(fs), "rpk redpanda mode", ) } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda.go b/src/go/rpk/pkg/cli/cmd/redpanda.go index 14d68a75320e4..e0c980a5cd3af 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda.go @@ -15,26 +15,23 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda/admin" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewRedpandaCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewRedpandaCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { command := &cobra.Command{ Use: "redpanda", Short: "Interact with a local Redpanda process", } - command.AddCommand(redpanda.NewStartCommand(fs, mgr, launcher)) - command.AddCommand(redpanda.NewStopCommand(fs, mgr)) + command.AddCommand(redpanda.NewStartCommand(fs, launcher)) + command.AddCommand(redpanda.NewStopCommand(fs)) command.AddCommand(redpanda.NewCheckCommand(fs)) command.AddCommand(redpanda.NewTuneCommand(fs)) - command.AddCommand(redpanda.NewModeCommand(mgr)) - command.AddCommand(redpanda.NewConfigCommand(fs, mgr)) + command.AddCommand(redpanda.NewModeCommand(fs)) + command.AddCommand(redpanda.NewConfigCommand(fs)) command.AddCommand(admin.NewCommand(fs)) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config.go b/src/go/rpk/pkg/cli/cmd/redpanda/config.go index b79a8d0f857a6..5ff0dbc952d11 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config.go @@ -19,52 +19,47 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" vnet "github.com/redpanda-data/redpanda/src/go/rpk/pkg/net" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" ) const configFileFlag = "config" -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { root := &cobra.Command{ Use: "config ", Short: "Edit configuration.", } - root.AddCommand(set(fs, mgr)) - root.AddCommand(bootstrap(mgr)) - root.AddCommand(initNode(mgr)) + root.AddCommand(set(fs)) + root.AddCommand(bootstrap(fs)) + root.AddCommand(initNode(fs)) return root } -func set(fs afero.Fs, mgr config.Manager) *cobra.Command { +func set(fs afero.Fs) *cobra.Command { var ( format string configPath string ) c := &cobra.Command{ Use: "set ", - Short: "Set configuration values, such as the node IDs or the list of seed servers.", + Short: "Set configuration values, such as the node IDs or the list of seed servers", Args: cobra.ExactArgs(2), - RunE: func(_ *cobra.Command, args []string) error { - var err error - key := args[0] - value := args[1] - if configPath == "" { - configPath, err = config.FindConfigFile(fs) - if err != nil { - return err - } - } - _, err = mgr.Read(configPath) + RunE: func(cmd *cobra.Command, args []string) error { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) if err != nil { - return err + return fmt.Errorf("unable to load config: %v", err) } - err = mgr.Set(key, value, format) + + err = cfg.Set(args[0], args[1], format) if err != nil { return err } - return mgr.WriteLoaded() + + return cfg.Write(fs) }, } c.Flags().StringVar(&format, @@ -84,7 +79,7 @@ func set(fs afero.Fs, mgr config.Manager) *cobra.Command { return c } -func bootstrap(mgr config.Manager) *cobra.Command { +func bootstrap(fs afero.Fs) *cobra.Command { var ( ips []string self string @@ -93,44 +88,36 @@ func bootstrap(mgr config.Manager) *cobra.Command { ) c := &cobra.Command{ Use: "bootstrap --id [--self ] [--ips ]", - Short: "Initialize the configuration to bootstrap a cluster.", + Short: "Initialize the configuration to bootstrap a cluster", Long: helpBootstrap, Args: cobra.OnlyValidArgs, - RunE: func(c *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + seeds, err := parseSeedIPs(ips) - if err != nil { - return err - } - var ownIP net.IP - if self != "" { - ownIP = net.ParseIP(self) - if ownIP == nil { - return fmt.Errorf("%s is not a valid IP", self) - } - } else { - ownIP, err = getOwnIP() - if err != nil { - return err - } - } - conf.Redpanda.ID = id - conf.Redpanda.RPCServer.Address = ownIP.String() - conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ + out.MaybeDieErr(err) + + ownIP, err := parseSelfIP(self) + out.MaybeDieErr(err) + + cfg.Redpanda.ID = id + cfg.Redpanda.RPCServer.Address = ownIP.String() + cfg.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultKafkaPort, }} - conf.Redpanda.AdminAPI = []config.NamedSocketAddress{{ + cfg.Redpanda.AdminAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultAdminPort, }} - conf.Redpanda.SeedServers = []config.SeedServer{} - conf.Redpanda.SeedServers = seeds - return mgr.Write(conf) + cfg.Redpanda.SeedServers = []config.SeedServer{} + cfg.Redpanda.SeedServers = seeds + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringSliceVar( @@ -162,22 +149,25 @@ func bootstrap(mgr config.Manager) *cobra.Command { return c } -func initNode(mgr config.Manager) *cobra.Command { +func initNode(fs afero.Fs) *cobra.Command { var configPath string c := &cobra.Command{ Use: "init", - Short: "Init the node after install, by setting the node's UUID.", + Short: "Init the node after install, by setting the node's UUID", Args: cobra.OnlyValidArgs, - RunE: func(_ *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + // Don't reset the node's UUID if it has already been set. - if conf.NodeUUID == "" { - return mgr.WriteNodeUUID(conf) + if cfg.NodeUUID == "" { + err = cfg.WriteNodeUUID() + out.MaybeDie(err, "error creating nodeUUID: %v", err) } - return nil + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringVar( @@ -190,6 +180,22 @@ func initNode(mgr config.Manager) *cobra.Command { return c } +func parseSelfIP(self string) (net.IP, error) { + if self != "" { + ownIP := net.ParseIP(self) + if ownIP == nil { + return nil, fmt.Errorf("%s is not a valid IP", self) + } + return ownIP, nil + } else { + ownIP, err := getOwnIP() + if err != nil { + return nil, err + } + return ownIP, nil + } +} + func parseSeedIPs(ips []string) ([]config.SeedServer, error) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port var seeds []config.SeedServer @@ -202,10 +208,8 @@ func parseSeedIPs(ips []string) ([]config.SeedServer, error) { host, port := vnet.SplitHostPortDefault(hostport, defaultRPCPort) seed := config.SeedServer{ - Host: config.SocketAddress{ - Address: host, - Port: port, - }, + Address: host, + Port: port, } seeds = append(seeds, seed) } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go index b69c0e536ac5b..943ab999998ee 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go @@ -13,166 +13,15 @@ package redpanda import ( - "path/filepath" "strings" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) -func TestSetCmd(t *testing.T) { - tests := []struct { - name string - key string - value string - args []string - expected interface{} - expectErr bool - }{ - { - name: "it should set single integer fields", - key: "redpanda.node_id", - value: "54312", - expected: 54312, - }, - { - name: "it should set single float fields", - key: "redpanda.float_field", - value: "42.3", - expected: 42.3, - }, - { - name: "it should set single string fields", - key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", - expected: "'/var/lib/differentdir'", - }, - { - name: "it should set single bool fields", - key: "rpk.enable_usage_stats", - value: "true", - expected: true, - }, - { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, - args: []string{"--format", "yaml"}, - expected: map[string]interface{}{ - "enable_usage_stats": false, - "overprovisioned": false, - "tune_network": false, - "tune_disk_scheduler": false, - "tune_disk_write_cache": false, - "tune_disk_nomerges": false, - "tune_disk_irq": true, - "tune_cpu": false, - "tune_aio_events": false, - "tune_clocksource": false, - "tune_swappiness": false, - "tune_transparent_hugepages": false, - "enable_memory_locking": false, - "tune_fstrim": false, - "tune_coredump": false, - "tune_ballast_file": false, - "coredump_dir": "/var/lib/redpanda/coredump", - }, - }, - { - name: "it should partially set map fields (json)", - key: "redpanda.kafka_api", - value: `[{ - "address": "192.168.54.2", - "port": 9092 -}]`, - args: []string{"--format", "json"}, - expected: []interface{}{ - map[interface{}]interface{}{ - "port": 9092, - "address": "192.168.54.2", - }, - }, - }, - { - name: "it should fail if the new value is invalid", - key: "redpanda", - value: `{"data_directory": ""}`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (json)", - key: "redpanda", - value: `{"seed_servers": []`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (yaml)", - key: "redpanda", - value: `seed_servers: -- host: - address: "123.`, - args: []string{"--format", "yaml"}, - expectErr: true, - }, - { - name: "it should fail if the format isn't supported", - key: "redpanda", - value: `node_id=1`, - args: []string{"--format", "toml"}, - expectErr: true, - }, - { - name: "it should fail if no key is passed", - value: `node_id=1`, - expectErr: true, - }, - { - name: "it should fail if no value is passed", - key: "rpk.tune_coredump", - expectErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - - c := NewConfigCommand(fs, mgr) - args := []string{"set"} - if tt.key != "" { - args = append(args, tt.key) - } - if tt.value != "" { - args = append(args, tt.value) - } - c.SetArgs(append(args, tt.args...)) - err = c.Execute() - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) - val := v.Get(tt.key) - require.Exactly(t, tt.expected, val) - }) - } -} - func TestBootstrap(t *testing.T) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port tests := []struct { @@ -193,22 +42,16 @@ func TestBootstrap(t *testing.T) { ips: []string{"187.89.76.3", "192.168.34.5", "192.168.45.8"}, expSeedServers: []config.SeedServer{ { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: defaultRPCPort, - }, + Address: "187.89.76.3", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "192.168.34.5", - Port: defaultRPCPort, - }, + Address: "192.168.34.5", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "192.168.45.8", - Port: defaultRPCPort, - }, + Address: "192.168.45.8", + Port: defaultRPCPort, }, }, self: "192.168.34.5", @@ -219,72 +62,36 @@ func TestBootstrap(t *testing.T) { ips: []string{"187.89.76.3", "localhost", "redpanda.com", "test-url.net", "187.89.76.3:80"}, expSeedServers: []config.SeedServer{ { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: defaultRPCPort, - }, + Address: "187.89.76.3", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "localhost", - Port: defaultRPCPort, - }, + Address: "localhost", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "redpanda.com", - Port: defaultRPCPort, - }, + Address: "redpanda.com", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "test-url.net", - Port: defaultRPCPort, - }, + Address: "test-url.net", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: 80, - }, + Address: "187.89.76.3", + Port: 80, }, }, self: "192.168.34.5", id: "1", }, - { - name: "it should fail if any of the --ips IPs isn't valid", - ips: []string{"187.89.9", "192.168.34.5", "192.168.45.8"}, - self: "192.168.34.5", - id: "1", - expectedErr: `invalid host "187.89.9" does not match "host", nor "host:port", nor "scheme://host:port"`, - }, - { - name: "it should fail if --self isn't a valid IP", - ips: []string{"187.89.9.78", "192.168.34.5", "192.168.45.8"}, - self: "www.host.com", - id: "1", - expectedErr: "www.host.com is not a valid IP", - }, - { - name: "it should fail if --id isn't passed", - expectedErr: "required flag(s) \"id\" not set", - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - configPath, err := filepath.Abs("./redpanda.yaml") - require.NoError(t, err) fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - err = fs.MkdirAll( - filepath.Dir(configPath), - 0o644, - ) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"bootstrap", "--config", configPath} + c := bootstrap(fs) + var args []string if len(tt.ips) != 0 { args = append( args, @@ -299,15 +106,15 @@ func TestBootstrap(t *testing.T) { args = append(args, "--id", tt.id) } c.SetArgs(args) - err = c.Execute() + err := c.Execute() if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return } require.NoError(t, err) - _, err = fs.Stat(configPath) + _, err = fs.Stat(config.Default().ConfigFile) require.NoError(t, err) - conf, err := mgr.Read(configPath) + conf, err := new(config.Params).Load(fs) require.NoError(t, err) require.Equal(t, tt.self, conf.Redpanda.RPCServer.Address) @@ -327,25 +134,37 @@ func TestBootstrap(t *testing.T) { } func TestInitNode(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"init"} - c.SetArgs(args) + for _, test := range []struct { + name string + prevID string + }{ + {name: "without UUID"}, + {name: "with UUID", prevID: "my_id"}, + } { + t.Run(test.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + c := config.Default() + if test.prevID != "" { + c.NodeUUID = test.prevID + } - err = c.Execute() - require.NoError(t, err) + bs, err := yaml.Marshal(c) + require.NoError(t, err) + err = afero.WriteFile(fs, c.ConfigFile, bs, 0o644) + require.NoError(t, err) + + cmd := initNode(fs) + err = cmd.Execute() + require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) + conf, err := new(config.Params).Load(fs) + require.NoError(t, err) - val := v.Get("node_uuid") - require.NotEmpty(t, val) + if test.prevID != "" { + require.Exactly(t, conf.NodeUUID, test.prevID) + } else { + require.NotEmpty(t, conf.NodeUUID) + } + }) + } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go index 0378d76fd86f1..5ef6408e2acbf 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go @@ -17,11 +17,13 @@ import ( "strings" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" log "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { var configFile string command := &cobra.Command{ Use: "mode ", @@ -33,9 +35,9 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { } return nil }, - RunE: func(_ *cobra.Command, args []string) error { - // Safe to access args[0] because it was validated in Args - return executeMode(mgr, configFile, args[0]) + Run: func(cmd *cobra.Command, args []string) { + err := executeMode(fs, cmd, args) + out.MaybeDieErr(err) }, } command.Flags().StringVar( @@ -48,15 +50,23 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { return command } -func executeMode(mgr config.Manager, configFile string, mode string) error { - conf, err := mgr.FindOrGenerate(configFile) +func executeMode(fs afero.Fs, cmd *cobra.Command, args []string) error { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + if err != nil { + return fmt.Errorf("unable to load config: %v", err) + } + // Safe to access args[0] because it was validated in Args + mode := args[0] + cfg, err = config.SetMode(mode, cfg) if err != nil { return err } - conf, err = config.SetMode(mode, conf) + + log.Infof("Writing '%s' mode defaults to '%s'", mode, cfg.ConfigFile) + err = cfg.Write(fs) if err != nil { return err } - log.Infof("Writing '%s' mode defaults to '%s'", mode, conf.ConfigFile) - return mgr.Write(conf) + return nil } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go index b65f4be4d1b21..1d2c99e88345e 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go @@ -13,17 +13,15 @@ package redpanda import ( - "bytes" "fmt" "os" - "strings" + "reflect" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func fillRpkConfig(path, mode string) *config.Config { @@ -50,18 +48,12 @@ func fillRpkConfig(path, mode string) *config.Config { func TestModeCommand(t *testing.T) { configPath := "/etc/redpanda/redpanda.yaml" - dir, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - wdConfigPath := fmt.Sprintf("%s/redpanda.yaml", dir) tests := []struct { name string args []string before func(afero.Fs) (string, error) expectedConfig *config.Config - expectedOutput string - expectedErrMsg string + expectedError bool }{ { name: "development mode should disable all fields in the rpk config", @@ -74,8 +66,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'development' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "production mode should enable all fields in the rpk config", @@ -88,8 +78,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'production' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "the development mode alias, 'dev', should work the same", @@ -102,8 +90,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'dev' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "the production mode alias, 'prod', should work the same", @@ -116,8 +102,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "mode should work if --config isn't passed, but the file is in /etc/redpanda/redpanda.yaml", @@ -130,46 +114,61 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "mode lists the available modes if the one passed is not valid", args: []string{"invalidmode"}, before: func(fs afero.Fs) (string, error) { + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + wdConfigPath := fmt.Sprintf("%s/redpanda.yaml", dir) bs, err := yaml.Marshal(fillRpkConfig(wdConfigPath, config.ModeDev)) if err != nil { return "", err } return wdConfigPath, afero.WriteFile(fs, wdConfigPath, bs, 0o644) }, - expectedOutput: "", - expectedErrMsg: "'invalidmode' is not a supported mode. Available modes: dev, development, prod, production", + expectedError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - path, err := tt.before(fs) + _, err := tt.before(fs) require.NoError(t, err) - var out bytes.Buffer - cmd := NewModeCommand(mgr) + cmd := NewModeCommand(fs) cmd.SetArgs(tt.args) - logrus.SetOutput(&out) - err = cmd.Execute() - if tt.expectedErrMsg != "" { - require.EqualError(t, err, tt.expectedErrMsg) + err = executeMode(fs, cmd, tt.args) + if tt.expectedError && err != nil { return } require.NoError(t, err) - output := out.String() - require.Contains(t, strings.TrimSpace(output), tt.expectedOutput) - mgr = config.NewManager(fs) - conf, err := mgr.Read(path) + + conf, err := new(config.Params).Load(fs) require.NoError(t, err) - require.Exactly(t, tt.expectedConfig, conf) + + require.Exactly(t, tt.expectedConfig.Redpanda.DeveloperMode, conf.Redpanda.DeveloperMode) + for _, tuner := range []string{ + "TuneNetwork", + "TuneDiskScheduler", + "TuneDiskWriteCache", + "TuneNomerges", + "TuneDiskIrq", + "TuneFstrim", + "TuneCPU", + "TuneAioEvents", + "TuneClocksource", + "TuneSwappiness", + "Overprovisioned", + "TuneBallastFile", + } { + e := reflect.ValueOf(tt.expectedConfig.Rpk).FieldByName(tuner) + v := reflect.ValueOf(conf.Rpk).FieldByName(tuner) + require.Exactly(t, e.Bool(), v.Bool()) + } }) } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start.go b/src/go/rpk/pkg/cli/cmd/redpanda/start.go index b362bfad50908..bdee9b07de878 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start.go @@ -113,9 +113,7 @@ func parseConfigKvs(args []string) ([]string, []string) { return kvs, args } -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { prestartCfg := prestartConfig{} var ( configFile string @@ -143,25 +141,26 @@ func NewStartCommand( // (POSIX standard) UnknownFlags: true, }, - RunE: func(ccmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { // --set flags have to be parsed by hand because pflag (the // underlying flag-parsing lib used by cobra) uses a CSV parser // for list flags, and since JSON often contains commas, it // blows up when there's a JSON object. configKvs, filteredArgs := parseConfigKvs(os.Args) - conf, err := mgr.FindOrGenerate(configFile) + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) if err != nil { - return err + return fmt.Errorf("unable to load config file: %s", err) } if len(configKvs) > 0 { - conf, err = setConfig(mgr, configKvs) + err = setConfig(cfg, configKvs) if err != nil { return err } } - updateConfigWithFlags(conf, ccmd.Flags()) + updateConfigWithFlags(cfg, cmd.Flags()) env := api.EnvironmentPayload{} if len(seeds) == 0 { @@ -177,11 +176,11 @@ func NewStartCommand( } seedServers, err := parseSeeds(seeds) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(seedServers) != 0 { - conf.Redpanda.SeedServers = seedServers + cfg.Redpanda.SeedServers = seedServers } kafkaAddr = stringSliceOr( @@ -196,11 +195,11 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(kafkaAPI) > 0 { - conf.Redpanda.KafkaAPI = kafkaAPI + cfg.Redpanda.KafkaAPI = kafkaAPI } proxyAddr = stringSliceOr( @@ -215,14 +214,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(proxyAPI) > 0 { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.PandaproxyAPI = proxyAPI + cfg.Pandaproxy.PandaproxyAPI = proxyAPI } schemaRegAddr = stringSliceOr( @@ -237,14 +236,14 @@ func NewStartCommand( config.DefaultSchemaRegPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(schemaRegAPI) > 0 { - if conf.SchemaRegistry == nil { - conf.SchemaRegistry = config.Default().SchemaRegistry + if cfg.SchemaRegistry == nil { + cfg.SchemaRegistry = config.Default().SchemaRegistry } - conf.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI + cfg.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI } rpcAddr = stringOr( @@ -256,11 +255,11 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if rpcServer != nil { - conf.Redpanda.RPCServer = *rpcServer + cfg.Redpanda.RPCServer = *rpcServer } advertisedKafka = stringSliceOr( @@ -275,11 +274,12 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - if advKafkaAPI != nil { - conf.Redpanda.AdvertisedKafkaAPI = advKafkaAPI + + if len(advKafkaAPI) > 0 { + cfg.Redpanda.AdvertisedKafkaAPI = advKafkaAPI } advertisedProxy = stringSliceOr( @@ -294,14 +294,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advProxyAPI != nil { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI + cfg.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI } advertisedRPC = stringOr( @@ -313,50 +313,50 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advRPCApi != nil { - conf.Redpanda.AdvertisedRPCAPI = advRPCApi + cfg.Redpanda.AdvertisedRPCAPI = advRPCApi } installDirectory, err := cli.GetOrFindInstallDir(fs, installDirFlag) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } rpArgs, err := buildRedpandaFlags( fs, - conf, + cfg, filteredArgs, sFlags, - ccmd.Flags(), + cmd.Flags(), !prestartCfg.checkEnabled, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } checkPayloads, tunerPayloads, err := prestart( fs, rpArgs, - conf, + cfg, prestartCfg, timeout, ) env.Checks = checkPayloads env.Tuners = tunerPayloads if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - err = mgr.Write(conf) + err = cfg.Write(fs) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, nil) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, nil) rpArgs.ExtraArgs = args log.Info(common.FeedbackMsg) log.Info("Starting redpanda...") @@ -640,21 +640,21 @@ func mergeFlags( return current } -func setConfig(mgr config.Manager, configKvs []string) (*config.Config, error) { +func setConfig(cfg *config.Config, configKvs []string) error { for _, rawKv := range configKvs { parts := strings.SplitN(rawKv, "=", 2) if len(parts) < 2 { - return nil, fmt.Errorf( + return fmt.Errorf( "key-value pair '%s' is not formatted as expected (k=v)", rawKv, ) } - err := mgr.Set(parts[0], parts[1], "") + err := cfg.Set(parts[0], parts[1], "") if err != nil { - return nil, err + return err } } - return mgr.Get() + return nil } func resolveWellKnownIo( @@ -873,7 +873,7 @@ func parseSeeds(seeds []string) ([]config.SeedServer, error) { } seedServers = append( seedServers, - config.SeedServer{Host: *addr}, + config.SeedServer{Address: addr.Address, Port: addr.Port}, ) } return seedServers, nil diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go index 5fb8afc148759..d34c46ada3bad 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go @@ -14,7 +14,9 @@ package redpanda import ( "bytes" + "net" "os" + "strconv" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" @@ -107,18 +109,10 @@ func TestParseSeeds(t *testing.T) { name: "it should parse well-formed seed addrs", arg: []string{"127.0.0.1:1234", "domain.com:9892", "lonely-host", "192.168.34.1"}, expected: []config.SeedServer{ - { - Host: config.SocketAddress{Address: "127.0.0.1", Port: 1234}, - }, - { - Host: config.SocketAddress{Address: "domain.com", Port: 9892}, - }, - { - Host: config.SocketAddress{Address: "lonely-host", Port: 33145}, - }, - { - Host: config.SocketAddress{Address: "192.168.34.1", Port: 33145}, - }, + {Address: "127.0.0.1", Port: 1234}, + {Address: "domain.com", Port: 9892}, + {Address: "lonely-host", Port: 33145}, + {Address: "192.168.34.1", Port: 33145}, }, }, { @@ -171,7 +165,7 @@ func TestStartCommand(t *testing.T) { 0o755, ) }, - expectedErrMsg: "An error happened while trying to read /etc/redpanda/redpanda.yaml: While parsing config: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml` into map[string]interface {}", + expectedErrMsg: "unable to load config file: unable to yaml decode /etc/redpanda/redpanda.yaml: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml`", }, { name: "should generate the config at the given path if it doesn't exist", args: []string{ @@ -191,11 +185,15 @@ func TestStartCommand(t *testing.T) { "The config should have been created at '%s'", path, ) - defaultConf := config.Default() - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + c := config.Default() + // Adding unset default that get added on first load. + b0 := c.Redpanda.KafkaAPI[0] + c.Rpk.KafkaAPI.Brokers = []string{net.JoinHostPort(b0.Address, strconv.Itoa(b0.Port))} + c.Rpk.AdminAPI.Addresses = []string{"127.0.0.1:9644"} + + conf, err := new(config.Params).Load(fs) require.NoError(st, err) - require.Exactly(st, defaultConf, conf) + require.Exactly(st, c, conf.File()) }, }, { name: "it should write the given config file path", @@ -208,8 +206,7 @@ func TestStartCommand(t *testing.T) { }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, path, conf.ConfigFile) }, @@ -255,9 +252,7 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -321,9 +316,7 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -375,9 +368,7 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // The value set through the --kafka-addr flag should // have been picked. @@ -396,9 +387,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -408,13 +397,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) - conf := config.Default() - return mgr.Write(conf) + cfg := config.Default() + return cfg.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -426,9 +413,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, 34, conf.Redpanda.ID) }, @@ -439,9 +424,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, config.Default().Redpanda.ID, conf.Redpanda.ID) @@ -452,14 +435,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.ID = 98 - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -475,9 +456,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, "aws:i3xlarge:default", conf.Rpk.WellKnownIo) }, @@ -488,14 +467,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.WellKnownIo = "gcp:n2standard:ssd" - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -513,9 +490,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, false, conf.Rpk.Overprovisioned) @@ -526,13 +501,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -549,9 +522,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, true, conf.Rpk.EnableMemoryLocking) @@ -563,14 +534,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.EnableMemoryLocking = true - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -586,24 +555,17 @@ func TestStartCommand(t *testing.T) { "--seeds", "192.168.34.32:33145,somehost:54321,justahostnoport", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "192.168.34.32", - Port: 33145, - }, + Address: "192.168.34.32", + Port: 33145, }, { - Host: config.SocketAddress{ - Address: "somehost", - Port: 54321, - }, + Address: "somehost", + Port: 54321, }, { - Host: config.SocketAddress{ - Address: "justahostnoport", - Port: 33145, - }, + Address: "justahostnoport", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -620,24 +582,17 @@ func TestStartCommand(t *testing.T) { "-s", "192.168.123.32:33146,host", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "192.168.3.32", - Port: 33145, - }, + Address: "192.168.3.32", + Port: 33145, }, { - Host: config.SocketAddress{ - Address: "192.168.123.32", - Port: 33146, - }, + Address: "192.168.123.32", + Port: 33146, }, { - Host: config.SocketAddress{ - Address: "host", - Port: 33145, - }, + Address: "host", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -659,19 +614,14 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_SEEDS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }, { - Host: config.SocketAddress{ - Address: "host", - Port: 33145, - }, + Address: "host", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -686,25 +636,19 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.SeedServers = []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }} // Check that the generated config is as expected. require.Exactly( @@ -732,8 +676,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -753,8 +696,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -787,8 +729,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "host", @@ -807,17 +748,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.RPCServer = config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.33.33", @@ -837,8 +776,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -858,8 +796,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -879,8 +816,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -901,8 +837,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32,host:9092", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -939,8 +874,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -959,17 +893,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -989,8 +921,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1010,8 +941,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1044,8 +974,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1064,17 +993,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedKafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -1094,8 +1021,7 @@ func TestStartCommand(t *testing.T) { "--advertise-pandaproxy-addr", "192.168.34.32:8083", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1121,8 +1047,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_PANDAPROXY_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1142,8 +1067,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1163,8 +1087,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1197,8 +1120,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "host", @@ -1217,17 +1139,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedRPCAPI = &config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.33.33", @@ -1246,10 +1166,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--overprovisioned", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--overprovisioned"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--overprovisioned' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--overprovisioned' directly to `rpk start`.", }, { @@ -1258,10 +1177,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--smp", "1", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--smp=1"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--smp' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--smp' directly to `rpk start`.", }, { @@ -1270,10 +1188,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--memory", "2G", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--memory=1G"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--memory' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--memory' directly to `rpk start`.", }, { @@ -1282,12 +1199,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--smp=3", "--smp=55", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1302,12 +1218,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--logger-log-level=archival=debug:cloud_storage=debug", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1362,7 +1277,6 @@ func TestStartCommand(t *testing.T) { defer tt.after() } fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) var launcher redpanda.Launcher = &noopLauncher{} if tt.launcher != nil { launcher = tt.launcher @@ -1372,11 +1286,11 @@ func TestStartCommand(t *testing.T) { } var out bytes.Buffer logrus.SetOutput(&out) - c := NewStartCommand(fs, mgr, launcher) + c := NewStartCommand(fs, launcher) c.SetArgs(tt.args) err := c.Execute() if tt.expectedErrMsg != "" { - require.EqualError(st, err, tt.expectedErrMsg) + require.Contains(st, err.Error(), tt.expectedErrMsg) return } require.NoError(st, err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go index 5f452ba823732..4168e5318f644 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go @@ -14,6 +14,7 @@ package redpanda import ( "errors" + "fmt" "strconv" "syscall" "time" @@ -26,7 +27,7 @@ import ( "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { var ( configFile string timeout time.Duration @@ -38,8 +39,8 @@ func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { first sends SIGINT, and waits for the specified timeout. Then, if redpanda hasn't stopped, it sends SIGTERM. Lastly, it sends SIGKILL if it's still running.`, - RunE: func(ccmd *cobra.Command, args []string) error { - return executeStop(fs, mgr, configFile, timeout) + RunE: func(cmd *cobra.Command, args []string) error { + return executeStop(fs, cmd, timeout) }, } command.Flags().StringVar( @@ -63,14 +64,14 @@ running.`, return command } -func executeStop( - fs afero.Fs, mgr config.Manager, configFile string, timeout time.Duration, -) error { - conf, err := mgr.ReadOrFind(configFile) +func executeStop(fs afero.Fs, cmd *cobra.Command, timeout time.Duration) error { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) if err != nil { - return err + return fmt.Errorf("unable to load config: %v", err) } - pidFile := conf.PIDFile() + + pidFile := cfg.PIDFile() isLocked, err := os.CheckLocked(pidFile) if err != nil { log.Debugf("error checking if the PID file is locked: %v", err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go index f6e4319ba4f38..7b7cc1a48be74 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go @@ -55,7 +55,6 @@ func TestStopCommand(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) conf := config.Default() command := baseCommand // trap the signals we want to ignore, to check that the @@ -76,11 +75,11 @@ func TestStopCommand(t *testing.T) { conf.PIDFile(), ) require.NoError(t, err) - err = mgr.Write(conf) + err = conf.Write(fs) require.NoError(t, err) var out bytes.Buffer - c := cmd.NewStopCommand(fs, mgr) + c := cmd.NewStopCommand(fs) args := append([]string{"--config", conf.ConfigFile}, tt.args...) c.SetArgs(args) diff --git a/src/go/rpk/pkg/cli/cmd/root.go b/src/go/rpk/pkg/cli/cmd/root.go index c8f83f450d518..947ed95fe0598 100644 --- a/src/go/rpk/pkg/cli/cmd/root.go +++ b/src/go/rpk/pkg/cli/cmd/root.go @@ -37,7 +37,6 @@ import ( func Execute() { verbose := false fs := afero.NewOsFs() - mgr := config.NewManager(fs) if !term.IsTerminal(int(os.Stdout.Fd())) { color.NoColor = true @@ -77,7 +76,7 @@ func Execute() { plugincmd.NewCommand(fs), ) - addPlatformDependentCmds(fs, mgr, root) + addPlatformDependentCmds(fs, root) // To support autocompletion even for plugins, we list all plugins now // and add tiny commands to our root command. Cobra works by creating diff --git a/src/go/rpk/pkg/cli/cmd/root_darwin.go b/src/go/rpk/pkg/cli/cmd/root_darwin.go index c2ad57b703d78..c16d212d62781 100644 --- a/src/go/rpk/pkg/cli/cmd/root_darwin.go +++ b/src/go/rpk/pkg/cli/cmd/root_darwin.go @@ -10,13 +10,10 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { cmd.AddCommand(NewRedpandaDarwinCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/root_linux.go b/src/go/rpk/pkg/cli/cmd/root_linux.go index c71ca1b26a9d7..8b66245fddf49 100644 --- a/src/go/rpk/pkg/cli/cmd/root_linux.go +++ b/src/go/rpk/pkg/cli/cmd/root_linux.go @@ -10,23 +10,20 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { - cmd.AddCommand(NewRedpandaCommand(fs, mgr, redpanda.NewLauncher())) +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { + cmd.AddCommand(NewRedpandaCommand(fs, redpanda.NewLauncher())) cmd.AddCommand(NewTuneCommand(fs)) cmd.AddCommand(NewCheckCommand(fs)) cmd.AddCommand(NewIoTuneCmd(fs)) - cmd.AddCommand(NewStartCommand(fs, mgr, redpanda.NewLauncher())) - cmd.AddCommand(NewStopCommand(fs, mgr)) - cmd.AddCommand(NewConfigCommand(fs, mgr)) + cmd.AddCommand(NewStartCommand(fs, redpanda.NewLauncher())) + cmd.AddCommand(NewStopCommand(fs)) + cmd.AddCommand(NewConfigCommand(fs)) cmd.AddCommand(NewStatusCommand(fs)) - cmd.AddCommand(NewModeCommand(mgr)) + cmd.AddCommand(NewModeCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/start.go b/src/go/rpk/pkg/cli/cmd/start.go index db83e7f97b6d4..dfe0251863aeb 100644 --- a/src/go/rpk/pkg/cli/cmd/start.go +++ b/src/go/rpk/pkg/cli/cmd/start.go @@ -15,17 +15,14 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { return common.Deprecated( - redpanda.NewStartCommand(fs, mgr, launcher), + redpanda.NewStartCommand(fs, launcher), "rpk redpanda start", ) } diff --git a/src/go/rpk/pkg/cli/cmd/stop.go b/src/go/rpk/pkg/cli/cmd/stop.go index 86adf9d278e08..7485c774adf0c 100644 --- a/src/go/rpk/pkg/cli/cmd/stop.go +++ b/src/go/rpk/pkg/cli/cmd/stop.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewStopCommand(fs, mgr), + redpanda.NewStopCommand(fs), "rpk redpanda stop", ) } diff --git a/src/go/rpk/pkg/config/config.go b/src/go/rpk/pkg/config/config.go index 5279fd1ea8c1f..b315affa32056 100644 --- a/src/go/rpk/pkg/config/config.go +++ b/src/go/rpk/pkg/config/config.go @@ -10,16 +10,10 @@ package config import ( - "errors" "fmt" - fp "path/filepath" "strings" - "github.com/mitchellh/mapstructure" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) const ( @@ -35,107 +29,35 @@ const ( DefaultBallastFileSize = "1GiB" ) -func InitViper(fs afero.Fs) *viper.Viper { - v := viper.New() - v.SetFs(fs) - v.SetConfigName("redpanda") - v.SetConfigType("yaml") - - // Viper does not take into account our explicit SetConfigType when - // calling ReadInConfig, instead it internally uses SupportedExts. - // Since we only ever want to load yaml, setting this global disables - // ReadInConfig from using any existing json files. - viper.SupportedExts = []string{"yaml"} - - setDefaults(v) - return v -} - -func addConfigPaths(v *viper.Viper) { - v.AddConfigPath("$HOME") - v.AddConfigPath(fp.Join("etc", "redpanda")) - v.AddConfigPath(".") -} - -func setDefaults(v *viper.Viper) { - var traverse func(tree map[string]interface{}, path ...string) - traverse = func(tree map[string]interface{}, path ...string) { - for key, val := range tree { - if subtree, ok := val.(map[string]interface{}); ok { - traverse(subtree, append(path, key)...) - } else { - v.SetDefault( - strings.Join(append(path, key), "."), - val, - ) - } - } - } - traverse(defaultMap()) -} - func Default() *Config { - conf := &Config{} - err := mapstructure.Decode(defaultMap(), conf) - if err != nil { - panic(err) - } - return conf -} - -func defaultMap() map[string]interface{} { - var defaultListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9092, - } - defaultListeners := []interface{}{defaultListener} - var defaultAdminListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9644, - } - defaultAdminListeners := []interface{}{defaultAdminListener} - return map[string]interface{}{ - "config_file": "/etc/redpanda/redpanda.yaml", - "pandaproxy": Pandaproxy{}, - "schema_registry": SchemaRegistry{}, - "redpanda": map[string]interface{}{ - "data_directory": "/var/lib/redpanda/data", - "rpc_server": map[string]interface{}{ - "address": "0.0.0.0", - "port": 33145, + return &Config{ + ConfigFile: "/etc/redpanda/redpanda.yaml", + Redpanda: RedpandaConfig{ + Directory: "/var/lib/redpanda/data", + RPCServer: SocketAddress{ + Address: "0.0.0.0", + Port: 33145, }, - "kafka_api": defaultListeners, - "admin": defaultAdminListeners, - "node_id": 0, - "seed_servers": []interface{}{}, - "developer_mode": true, + KafkaAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9092, + }}, + AdminAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9644, + }}, + SeedServers: []SeedServer{}, + DeveloperMode: true, }, - "rpk": map[string]interface{}{ - "coredump_dir": "/var/lib/redpanda/coredump", + Rpk: RpkConfig{ + CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } } -func findBackup(fs afero.Fs, dir string) (string, error) { - exists, err := afero.Exists(fs, dir) - if err != nil { - return "", err - } - if !exists { - return "", nil - } - files, err := afero.ReadDir(fs, dir) - if err != nil { - return "", err - } - for _, f := range files { - if strings.HasSuffix(f.Name(), ".bk") { - return fmt.Sprintf("%s/%s", dir, f.Name()), nil - } - } - return "", nil -} - func SetMode(mode string, conf *Config) (*Config, error) { m, err := NormalizeMode(mode) if err != nil { @@ -223,89 +145,52 @@ func AvailableModes() []string { } } -func Check(conf *Config) (bool, []error) { - configMap, err := toMap(conf) - if err != nil { - return false, []error{err} - } - - v := viper.New() - err = v.MergeConfigMap(configMap) - if err != nil { - return false, []error{err} - } - return check(v) -} - -func check(v *viper.Viper) (bool, []error) { - errs := checkRedpandaConfig(v) +func Check(cfg *Config) (bool, []error) { + errs := checkRedpandaConfig(cfg) errs = append( errs, - checkRpkConfig(v)..., + checkRpkConfig(cfg)..., ) ok := len(errs) == 0 return ok, errs } -func checkRedpandaConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetString("redpanda.data_directory") == "" { +func checkRedpandaConfig(cfg *Config) []error { + var errs []error + rp := cfg.Redpanda + // top level check + if rp.Directory == "" { errs = append(errs, fmt.Errorf("redpanda.data_directory can't be empty")) } - if v.GetInt("redpanda.node_id") < 0 { + if rp.ID < 0 { errs = append(errs, fmt.Errorf("redpanda.node_id can't be a negative integer")) } - rpcServerKey := "redpanda.rpc_server" - exists := v.Sub(rpcServerKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", rpcServerKey), - ) + // rpc server + if rp.RPCServer == (SocketAddress{}) { + errs = append(errs, fmt.Errorf("redpanda.rpc_server missing")) } else { - socket := &SocketAddress{} - err := unmarshalKey(v, rpcServerKey, socket) + _, err := yaml.Marshal(rp.RPCServer) if err != nil { - errs = append( - errs, - fmt.Errorf("invalid structure for %s", rpcServerKey), - ) + errs = append(errs, fmt.Errorf("invalid structure for redpanda.rpc_server")) } else { errs = append( errs, - checkSocketAddress(*socket, rpcServerKey)..., + checkSocketAddress(rp.RPCServer, "redpanda.rpc_server")..., ) } } - kafkaAPIKey := "redpanda.kafka_api" - exists = v.Get(kafkaAPIKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", kafkaAPIKey), - ) + // kafka api + if len(rp.KafkaAPI) == 0 { + errs = append(errs, fmt.Errorf("redpanda.kafka_api missing")) } else { - var kafkaListeners []NamedSocketAddress - err := unmarshalKey(v, "redpanda.kafka_api", &kafkaListeners) + _, err := yaml.Marshal(rp.KafkaAPI) if err != nil { - log.Error(err) - err = fmt.Errorf( - "%s doesn't have the expected structure", - kafkaAPIKey, - ) - return append( - errs, - err, - ) + return append(errs, fmt.Errorf("redpanda.kafka_api doesn't have the expected structure")) } - for i, addr := range kafkaListeners { - configPath := fmt.Sprintf( - "%s.%d", - kafkaAPIKey, - i, - ) + for i, addr := range rp.KafkaAPI { + configPath := fmt.Sprintf("redpanda.kafka_api.%d", i) errs = append( errs, checkSocketAddress( @@ -316,28 +201,18 @@ func checkRedpandaConfig(v *viper.Viper) []error { } } - var seedServersSlice []*SeedServer // map[string]interface{} - err := unmarshalKey(v, "redpanda.seed_servers", &seedServersSlice) - if err != nil { - log.Error(err) - msg := "redpanda.seed_servers doesn't have the expected structure" - return append( - errs, - errors.New(msg), - ) - } - if len(seedServersSlice) > 0 { - seedServersPath := "redpanda.seed_servers" - for i, seed := range seedServersSlice { - configPath := fmt.Sprintf( - "%s.%d.host", - seedServersPath, - i, - ) + // seed servers + if len(rp.SeedServers) > 0 { + _, err := yaml.Marshal(rp.KafkaAPI) + if err != nil { + return append(errs, fmt.Errorf("redpanda.seed_servers doesn't have the expected structure")) + } + for i, seed := range rp.SeedServers { + configPath := fmt.Sprintf("redpanda.seed_servers.%d.host", i) errs = append( errs, checkSocketAddress( - seed.Host, + SocketAddress(seed), configPath, )..., ) @@ -346,6 +221,14 @@ func checkRedpandaConfig(v *viper.Viper) []error { return errs } +func checkRpkConfig(cfg *Config) []error { + var errs []error + if cfg.Rpk.TuneCoredump && cfg.Rpk.CoredumpDir == "" { + errs = append(errs, fmt.Errorf("if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty")) + } + return errs +} + func checkSocketAddress(s SocketAddress, configPath string) []error { errs := []error{} if s.Port == 0 { @@ -356,62 +239,3 @@ func checkSocketAddress(s SocketAddress, configPath string) []error { } return errs } - -func checkRpkConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetBool("rpk.tune_coredump") && v.GetString("rpk.coredump_dir") == "" { - msg := "if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty" - errs = append(errs, errors.New(msg)) - } - return errs -} - -func decoderConfig() mapstructure.DecoderConfig { - return mapstructure.DecoderConfig{ - // Sometimes viper will save int values as strings (i.e. - // through BindPFlag) so we have to allow mapstructure - // to cast them. - WeaklyTypedInput: true, - DecodeHook: mapstructure.ComposeDecodeHookFunc( - // These 2 hooks are viper's default hooks. - // https://github.com/spf13/viper/blob/fb4eafdd9775508c450b90b1b72affeef4a68cf5/viper.go#L1004-L1005 - // They're set here because when decoderConfigOptions' resulting - // viper.DecoderConfigOption is used, viper's hooks are overridden. - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - // This hook translates the pre-21.1.4 configuration format to the - // latest one (see schema.go) - v21_1_4MapToNamedSocketAddressSlice, - // This hook translates the pre-21.4.1 TLS configuration format to the - // latest one (see schema.go) - v21_4_1TlsMapToNamedTlsSlice, - ), - } -} - -func decoderConfigOptions() viper.DecoderConfigOption { - return func(c *mapstructure.DecoderConfig) { - cfg := decoderConfig() - c.DecodeHook = cfg.DecodeHook - c.WeaklyTypedInput = cfg.WeaklyTypedInput - } -} - -func unmarshalKey(v *viper.Viper, key string, val interface{}) error { - return v.UnmarshalKey( - key, - val, - decoderConfigOptions(), - ) -} - -func toMap(conf *Config) (map[string]interface{}, error) { - mapConf := make(map[string]interface{}) - bs, err := yaml.Marshal(conf) - if err != nil { - return nil, err - } - err = yaml.Unmarshal(bs, &mapConf) - return mapConf, err -} diff --git a/src/go/rpk/pkg/config/config_test.go b/src/go/rpk/pkg/config/config_test.go index 6d7d891090bab..8e3bda73ac60e 100644 --- a/src/go/rpk/pkg/config/config_test.go +++ b/src/go/rpk/pkg/config/config_test.go @@ -10,25 +10,18 @@ package config import ( - "path/filepath" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - vyaml "github.com/redpanda-data/redpanda/src/go/rpk/pkg/yaml" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func getValidConfig() *Config { conf := Default() conf.Redpanda.SeedServers = []SeedServer{ - { - SocketAddress{"127.0.0.1", 33145}, - }, - { - SocketAddress{"127.0.0.1", 33146}, - }, + {"127.0.0.1", 33145}, + {"127.0.0.1", 33146}, } conf.Redpanda.DeveloperMode = false conf.Rpk = RpkConfig{ @@ -59,125 +52,107 @@ func TestSet(t *testing.T) { key string value string format string - check func(st *testing.T, c *Config, mgr *manager) + check func(st *testing.T, c *Config) expectErr bool }{ { - name: "it should parse '1' as an int and not as bool (true)", + name: "parse '1' as an int and not as bool (true)", key: "redpanda.node_id", value: "1", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 1, c.Redpanda.ID) }, }, { - name: "it should set single integer fields", + name: "set single integer fields", key: "redpanda.node_id", value: "54312", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should detect single integer fields if format isn't passed", + name: "detect single integer fields if format isn't passed", key: "redpanda.node_id", value: "54312", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should set single float fields", + name: "set single float fields", key: "redpanda.float_field", value: "42.3", format: "single", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should detect single float fields if format isn't passed", + name: "detect single float fields if format isn't passed", key: "redpanda.float_field", value: "42.3", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should set single string fields", + name: "set single string fields", key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", + value: "/var/lib/differentdir", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "'/var/lib/differentdir'", c.Redpanda.Directory) + check: func(st *testing.T, c *Config) { + require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should detect single string fields if format isn't passed", + name: "detect single string fields if format isn't passed", key: "redpanda.data_directory", value: "/var/lib/differentdir", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should set single bool fields", + name: "set single bool fields", key: "rpk.enable_usage_stats", value: "true", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should detect single bool fields if format isn't passed", + name: "detect single bool fields if format isn't passed", key: "rpk.enable_usage_stats", value: "true", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, + name: "partially set map fields (yaml)", + key: "rpk", + value: `tune_disk_irq: true +tune_cpu: true`, format: "yaml", - check: func(st *testing.T, c *Config, _ *manager) { - expected := RpkConfig{ - EnableUsageStats: false, - Overprovisioned: false, - TuneNetwork: false, - TuneDiskScheduler: false, - TuneNomerges: false, - TuneDiskIrq: true, - TuneCPU: false, - TuneAioEvents: false, - TuneClocksource: false, - TuneSwappiness: false, - TuneTransparentHugePages: false, - EnableMemoryLocking: false, - TuneFstrim: false, - TuneCoredump: false, - TuneDiskWriteCache: false, - CoredumpDir: "/var/lib/redpanda/coredump", - } - require.Exactly(st, expected, c.Rpk) + check: func(st *testing.T, c *Config) { + require.Exactly(st, true, c.Rpk.TuneDiskIrq) + require.Exactly(st, true, c.Rpk.TuneCPU) }, }, { - name: "it should detect pandaproxy client single field if format isn't passed", + name: "detect pandaproxy client single field if format isn't passed", key: "pandaproxy_client.retries", value: "42", - check: func(st *testing.T, c *Config, mgr *manager) { - require.Exactly(st, 42.0, c.PandaproxyClient.Other["retries"]) + check: func(st *testing.T, c *Config) { + require.Exactly(st, 42, c.PandaproxyClient.Other["retries"]) }, }, { - name: "it should detect yaml-formatted values if format isn't passed", + name: "detect yaml-formatted values if format isn't passed", key: "redpanda.kafka_api", value: `- name: external address: 192.168.73.45 @@ -186,7 +161,7 @@ func TestSet(t *testing.T) { address: 10.21.34.58 port: 9092 `, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Name: "external", Address: "192.168.73.45", @@ -200,14 +175,14 @@ func TestSet(t *testing.T) { }, }, { - name: "it should partially set map fields (json)", + name: "partially set map fields (json)", key: "redpanda.kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, format: "json", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -216,13 +191,13 @@ func TestSet(t *testing.T) { }, }, { - name: "it should detect json-formatted values if format isn't passed", + name: "detect json-formatted values if format isn't passed", key: "redpanda.advertised_kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -231,14 +206,22 @@ func TestSet(t *testing.T) { }, }, { - name: "it should fail if the value isn't well formatted (json)", + name: "set value of a slice", + key: "redpanda.admin.port", + value: "9641", + check: func(st *testing.T, c *Config) { + require.Exactly(st, 9641, c.Redpanda.AdminAPI[0].Port) + }, + }, + { + name: "fail if the value isn't well formatted (json)", key: "redpanda", value: `{"seed_servers": []`, format: "json", expectErr: true, }, { - name: "it should fail if the value isn't well formatted (yaml)", + name: "fail if the value isn't well formatted (yaml)", key: "redpanda", value: `seed_servers: - host: @@ -247,96 +230,38 @@ func TestSet(t *testing.T) { expectErr: true, }, { - name: "it should fail if the format isn't supported", + name: "fail if the format isn't supported", key: "redpanda", value: "node_id=1", format: "toml", expectErr: true, }, { - name: "it should fail if no key is passed", + name: "fail if no key is passed", value: `node_id=1`, expectErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set(tt.key, tt.value, tt.format) - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) - } - }) - } -} - -func TestMerge(t *testing.T) { - tests := []struct { - name string - config Config - check func(st *testing.T, c *Config, mgr *manager) - expectErr bool - }{ { - name: "it should merge Kafka API spec", - config: Config{ - Redpanda: RedpandaConfig{ - KafkaAPI: []NamedSocketAddress{{ - Name: "kafka-api-name", - Address: "1.2.3.4", - Port: 9123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "kafka-api-name", c.Redpanda.KafkaAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Redpanda.KafkaAPI[0].Address) - require.Exactly(st, 9123, c.Redpanda.KafkaAPI[0].Port) - }, - }, - { - name: "it should merge Pandaproxy API spec", - config: Config{ - Pandaproxy: &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{{ - Name: "proxy-api-name", - Address: "1.2.3.4", - Port: 8123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "proxy-api-name", c.Pandaproxy.PandaproxyAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Pandaproxy.PandaproxyAPI[0].Address) - require.Exactly(st, 8123, c.Pandaproxy.PandaproxyAPI[0].Port) - }, + name: "fail if deep unrecognized value is passed", + key: "redpanda.unrecognized.name", + value: "foo", + expectErr: true, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Merge(&tt.config) + cfg, err := new(Params).Load(fs) + require.NoError(t, err) + err = cfg.Set(tt.key, tt.value, tt.format) if tt.expectErr { require.Error(t, err) return } require.NoError(t, err) if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) + tt.check(t, cfg) } }) } @@ -370,53 +295,6 @@ func TestDefault(t *testing.T) { require.Exactly(t, expected, defaultConfig) } -func TestRead(t *testing.T) { - const baseDir string = "/etc/redpanda" - tests := []struct { - name string - path string - before func(afero.Fs, string) error - want func() *Config - wantErr bool - }{ - { - name: "shall return a config struct filled with values from the file", - before: func(fs afero.Fs, path string) error { - bs, err := yaml.Marshal(getValidConfig()) - if err != nil { - return err - } - if err = fs.MkdirAll(baseDir, 0o755); err != nil { - return err - } - _, err = utils.WriteBytes(fs, bs, path) - return err - }, - path: baseDir + "/redpanda.yaml", - want: getValidConfig, - wantErr: false, - }, - // TODO: Add tests for when the config file has missing objects - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := tt.before(fs, tt.path) - require.NoError(t, err) - got, err := mgr.Read(tt.path) - want := tt.want() - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.True(t, filepath.IsAbs(got.ConfigFile)) - require.Exactly(t, want, got) - }) - } -} - func TestWrite(t *testing.T) { tests := []struct { name string @@ -426,103 +304,52 @@ func TestWrite(t *testing.T) { expected string }{ { - name: "it should write the default values for redpanda.kafka_api if it's null", - conf: getValidConfig, - wantErr: false, + name: "write default values", + conf: getValidConfig, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "it should write the default values if redpanda.kafka_api is missing", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without advertised_kafka_api", + name: "write additional values", conf: func() *Config { c := getValidConfig() c.Redpanda.AdvertisedRPCAPI = &SocketAddress{ @@ -531,114 +358,53 @@ schema_registry: {} } return c }, - wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_rpc_api: - address: 174.32.64.2 - port: 33145 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + advertised_rpc_api: + address: 174.32.64.2 + port: 33145 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file without advertised_rpc_api", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdvertisedKafkaAPI = []NamedSocketAddress{{ - Address: "174.32.64.2", - Port: 9092, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_kafka_api: - - address: 174.32.64.2 - port: 9092 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without an rpk config object", + name: "write if empty struct is passed", conf: func() *Config { c := getValidConfig() c.Rpk = RpkConfig{} @@ -646,393 +412,47 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false -schema_registry: {} -`, - }, - { - name: "shall fail with an invalid config", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.Directory = "" - return c - }, - wantErr: true, - }, - { - name: "shall write a valid config file with an rpk config object", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall leave unrecognized fields untouched", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - node_id: 1 - - host: - address: 127.0.0.1 - port: 33146 - node_id: 2 - target_quota_byte_rate: 1000000 -unrecognized_top_field: - child: true -`, - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -unrecognized_top_field: - child: true -`, - }, - { - name: "should merge the new config onto the current one, not just overwrite it", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: func() *Config { - conf := getValidConfig() - conf.Redpanda.SeedServers = []SeedServer{} - conf.Redpanda.Directory = "/different/path" - return conf - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /different/path - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.KafkaAPI[0].Name = "outside" - c.Redpanda.KafkaAPITLS = []ServerTLS{{ - Name: "outside", - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - Other: map[string]interface{}{"principal_mapping_rules": "DEFAULT"}, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - name: outside - port: 9092 - kafka_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - name: outside - principal_mapping_rules: DEFAULT - require_client_auth: true - truststore_file: /etc/certs/ca.crt - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with admin tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdminAPITLS = []ServerTLS{{ - KeyFile: "/etc/certs/admin/cert.key", - TruststoreFile: "/etc/certs/admin/ca.crt", - CertFile: "/etc/certs/admin/cert.crt", - Enabled: true, - RequireClientAuth: true, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: false + tune_network: false + tune_disk_scheduler: false + tune_disk_nomerges: false + tune_disk_write_cache: false + tune_disk_irq: false + tune_fstrim: false + tune_cpu: false + tune_aio_events: false + tune_clocksource: false + tune_swappiness: false + tune_transparent_hugepages: false + enable_memory_locking: false + tune_coredump: false + tune_ballast_file: false + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_tls: - - cert_file: /etc/certs/admin/cert.crt - enabled: true - key_file: /etc/certs/admin/cert.key - require_client_auth: true - truststore_file: /etc/certs/admin/ca.crt - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write config with log_segment_size configuration", + name: "write unrecognized values ('Other' map).", conf: func() *Config { c := getValidConfig() size := 536870912 @@ -1044,469 +464,45 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - log_segment_size: 536870912 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false + log_segment_size: 536870912 rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with full pandaproxy configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - PandaproxyAPITLS: []ServerTLS{{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }}, - AdvertisedPandaproxyAPI: []NamedSocketAddress{ - { - Name: "advertised", - Address: "2.3.4.1", - Port: 2341, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - advertised_pandaproxy_api: - - address: 2.3.4.1 - name: advertised - port: 2341 - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 - pandaproxy_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy api only configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy client configuration", - conf: func() *Config { - c := getValidConfig() - c.PandaproxyClient = &KafkaClient{ - Brokers: []SocketAddress{ - { - Address: "1.2.3.4", - Port: 1234, - }, - }, - BrokerTLS: ServerTLS{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -pandaproxy_client: - broker_tls: - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt - brokers: - - address: 1.2.3.4 - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandproxy client SASL mechanism and SCRAM credentials", - conf: func() *Config { - c := getValidConfig() - mechanism := "abc" - username := "user" - password := "pass" - c.PandaproxyClient = &KafkaClient{ - SASLMechanism: &mechanism, - SCRAMUsername: &username, - SCRAMPassword: &password, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -pandaproxy_client: - sasl_mechanism: abc - scram_password: pass - scram_username: user -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file with scram configured", - conf: func() *Config { - c := getValidConfig() - c.Rpk.KafkaAPI.SASL = &SASL{ - User: "scram_user", - Password: "scram_password", - Mechanism: "SCRAM-SHA-256", - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - kafka_api: - sasl: - password: scram_password - type: SCRAM-SHA-256 - user: scram_user - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "should update an existing config with single kafka_api & advertised_kafka_api obj to a list", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - address: 0.0.0.0 - port: 9092 - advertised_kafka_api: - address: 1.cluster.redpanda.io - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: Default, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - advertised_kafka_api: - - address: 1.cluster.redpanda.io - port: 9092 - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false schema_registry: {} `, }, @@ -1515,19 +511,12 @@ schema_registry: {} t.Run(tt.name, func(t *testing.T) { path := tt.conf().ConfigFile fs := afero.NewMemMapFs() - mgr := NewManager(fs) if tt.existingConf != "" { _, err := utils.WriteBytes(fs, []byte(tt.existingConf), path) require.NoError(t, err) - } else { - // Write the config file so that WriteConfig backs it up - // when the new one is written. - err := vyaml.Persist(fs, tt.conf(), path) - require.NoError(t, err) } - _, err := mgr.Read(path) - require.NoError(t, err) - err = mgr.Write(tt.conf()) + + err := tt.conf().Write(fs) if tt.wantErr { require.Error(t, err) return @@ -1538,96 +527,6 @@ schema_registry: {} require.NoError(t, err) content := string(contentBytes) require.Equal(t, tt.expected, content) - backup, err := findBackup(fs, filepath.Dir(path)) - require.NoError(t, err) - _, err = fs.Stat(backup) - require.NoError(t, err) - }) - } -} - -func TestWriteLoaded(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set( - "rpk.admin", - `[{"address": "192.168.54.2","port": 9092}]`, - "json", - ) - require.NoError(t, err) - - mgr.WriteLoaded() - conf, err := mgr.Get() - require.NoError(t, err) - - newMgr := NewManager(fs) - newConf, err := newMgr.Read(Default().ConfigFile) - require.NoError(t, err) - - require.Exactly(t, conf, newConf) -} - -func TestReadOrGenerate(t *testing.T) { - tests := []struct { - name string - setup func(afero.Fs) error - configFile string - check func(st *testing.T, conf *Config) - expectError bool - }{ - { - name: "it should generate a config file at the given location", - configFile: Default().ConfigFile, - }, - { - name: "it shouldn't fail if there's a config file already", - setup: func(fs afero.Fs) error { - conf := Default() - mgr := NewManager(fs) - return mgr.Write(conf) - }, - configFile: Default().ConfigFile, - }, - { - name: "it should fail if the existing config file's content isn't valid yaml", - setup: func(fs afero.Fs) error { - bs := []byte(`redpanda: -- something`) - return vyaml.Persist(fs, bs, Default().ConfigFile) - }, - configFile: Default().ConfigFile, - expectError: true, - }, - { - name: "it should set config_file to the right value", - configFile: "./redpanda.yaml", - check: func(st *testing.T, conf *Config) { - path, err := filepath.Abs("./redpanda.yaml") - require.NoError(st, err) - require.Exactly(st, conf.ConfigFile, path) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - if tt.setup != nil { - err := tt.setup(fs) - require.NoError(t, err) - } - _, err := readOrGenerate(fs, InitViper(fs), tt.configFile) - if tt.expectError { - require.Error(t, err) - return - } - require.NoError(t, err) - conf, err := mgr.Read(tt.configFile) - require.NoError(t, err) - if tt.check != nil { - tt.check(t, conf) - } }) } } @@ -1816,7 +715,7 @@ func TestCheckConfig(t *testing.T) { name: "shall return an error when one of the seed servers' address is empty", conf: func() *Config { c := getValidConfig() - c.Redpanda.SeedServers[0].Host.Address = "" + c.Redpanda.SeedServers[0].Address = "" return c }, expected: []string{"redpanda.seed_servers.0.host.address can't be empty"}, @@ -1825,7 +724,7 @@ func TestCheckConfig(t *testing.T) { name: "shall return an error when one of the seed servers' port is 0", conf: func() *Config { c := getValidConfig() - c.Redpanda.SeedServers[1].Host.Port = 0 + c.Redpanda.SeedServers[1].Port = 0 return c }, expected: []string{"redpanda.seed_servers.1.host.port can't be 0"}, @@ -1849,8 +748,7 @@ func TestCheckConfig(t *testing.T) { c.Rpk.CoredumpDir = "" return c }, - expected: []string{"if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty"}, + expected: []string{"if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty"}, }, { name: "shall return no error if setup is empty," + @@ -1874,31 +772,3 @@ func TestCheckConfig(t *testing.T) { }) } } - -func TestWriteAndGenerateNodeUuid(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - baseDir := "/etc/redpanda" - path := baseDir + "/redpanda.yaml" - conf := getValidConfig() - conf.ConfigFile = path - bs, err := yaml.Marshal(conf) - require.NoError(t, err) - err = fs.MkdirAll(baseDir, 0o755) - require.NoError(t, err) - _, err = utils.WriteBytes(fs, bs, path) - require.NoError(t, err) - err = mgr.WriteNodeUUID(conf) - require.NoError(t, err) - require.NotEqual(t, "", conf.NodeUUID) - readConf, err := mgr.Read(path) - require.NoError(t, err) - require.Exactly(t, conf, readConf) -} - -func TestGet(t *testing.T) { - mgr := NewManager(afero.NewMemMapFs()) - conf, err := mgr.Get() - require.NoError(t, err) - require.Exactly(t, Default(), conf) -} diff --git a/src/go/rpk/pkg/config/find.go b/src/go/rpk/pkg/config/find.go deleted file mode 100644 index 4b2147b72b3f6..0000000000000 --- a/src/go/rpk/pkg/config/find.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" -) - -func FindConfigFile(fs afero.Fs) (string, error) { - log.Debugf("Looking for the redpanda config file") - configPathProviders := []func() ([]string, error){ - currentDirectory, - sysConfDirectory, - homeDirectory, - } - - lookedUpPaths := []string{} - for _, provider := range configPathProviders { - paths, err := provider() - if err != nil { - return "", err - } - for _, path := range paths { - candidate := filepath.Join(path, "redpanda.yaml") - lookedUpPaths = append(lookedUpPaths, candidate) - log.Debugf("Looking for redpanda config file in '%s'", path) - if exists, _ := afero.Exists(fs, candidate); exists { - return candidate, nil - } - } - } - // os.PathError can be checked with os.IsNotExist. - return "", &os.PathError{ - Op: "Open", - Path: strings.Join(lookedUpPaths, ", "), - Err: os.ErrNotExist, - } -} - -func sysConfDirectory() ([]string, error) { - return []string{"/etc/redpanda"}, nil -} - -func currentDirectory() ([]string, error) { - currentDir, err := os.Getwd() - if err != nil { - return nil, err - } - return []string{currentDir}, nil -} - -func homeDirectory() ([]string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, err - } - return []string{homeDir}, nil -} diff --git a/src/go/rpk/pkg/config/find_test.go b/src/go/rpk/pkg/config/find_test.go deleted file mode 100644 index bf6dbeb730582..0000000000000 --- a/src/go/rpk/pkg/config/find_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestFindConfig(t *testing.T) { - tests := []struct { - name string - before func(fs afero.Fs) - want string - wantErr bool - }{ - { - name: "should return an error when config is not found", - before: func(afero.Fs) {}, - want: "", - wantErr: true, - }, - { - name: "should return config file from home directory", - before: func(fs afero.Fs) { - createConfigIn(fs, homeDir()) - }, - want: filepath.Join(homeDir(), "redpanda.yaml"), - }, - { - name: "should return config file from 'etc' directory", - before: func(fs afero.Fs) { - createConfigIn(fs, filepath.Join("/", "etc", "redpanda")) - }, - want: filepath.Join("/", "etc", "redpanda", "redpanda.yaml"), - }, - { - name: "should return config file from current directory", - before: func(fs afero.Fs) { - createConfigIn(fs, currentDir()) - }, - want: filepath.Join(currentDir(), "redpanda.yaml"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - tt.before(fs) - got, err := FindConfigFile(fs) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, tt.want, got) - }) - } -} - -func createConfigIn(fs afero.Fs, path string) { - fs.Create(filepath.Join(path, "redpanda.yaml")) -} - -func currentDir() string { - d, _ := os.Getwd() - return d -} - -func homeDir() string { - d, _ := os.UserHomeDir() - return d -} diff --git a/src/go/rpk/pkg/config/license.go b/src/go/rpk/pkg/config/license.go deleted file mode 100644 index 5ecf38d923de0..0000000000000 --- a/src/go/rpk/pkg/config/license.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "hash/crc32" - "time" - - log "github.com/sirupsen/logrus" -) - -type LicenseKey struct { - Organization string `json:"o"` - ExpirationYear uint16 `json:"y"` - ExpirationMonth uint8 `json:"m"` - ExpirationDay uint8 `json:"d"` - Checksum uint32 `json:"c"` -} - -// Checks the key and prints a warning if the key is invalid. -func CheckAndPrintNotice(key string) { - err := CheckLicenseKey(key) - if err != nil { - log.Info(err) - return - } -} - -// Checks the license key, which is a Base64-encoded JSON object with the the -// organization it was granted to, the expiration date, and the hash of the -// concatenation of those fields. -// Returns true if the exp. date hasn't passed yet. Returns false otherwise. -// Returns an error if the key isn't base64-encoded or if it's not valid JSON. -func CheckLicenseKey(key string) error { - msg := "Please get a new one at https://vectorized.io/download-trial/" - if key == "" { - return errors.New("Missing license key. " + msg) - } - decoded, err := base64.StdEncoding.DecodeString(key) - invalidErr := errors.New("Invalid license key. " + msg) - if err != nil { - return invalidErr - } - lk := &LicenseKey{} - err = json.Unmarshal(decoded, lk) - if err != nil { - return invalidErr - } - - content := fmt.Sprintf( - "%s%d%d%d", - lk.Organization, - lk.ExpirationYear, - lk.ExpirationMonth, - lk.ExpirationDay, - ) - - checksum := crc32.ChecksumIEEE([]byte(content)) - - if lk.Checksum != checksum { - return invalidErr - } - - expDate := time.Date( - int(lk.ExpirationYear), - time.Month(lk.ExpirationMonth), - int(lk.ExpirationDay), - 0, 0, 0, 0, - time.UTC, - ) - - if time.Now().After(expDate) { - return errors.New("Your license key has expired. " + msg) - } - return nil -} diff --git a/src/go/rpk/pkg/config/license_test.go b/src/go/rpk/pkg/config/license_test.go deleted file mode 100644 index 20b5fd9df80de..0000000000000 --- a/src/go/rpk/pkg/config/license_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config_test - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "hash/crc32" - "testing" - "time" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/stretchr/testify/require" -) - -const vectorizedOrg string = "vectorized.io" - -func makeKey(org string, exp time.Time, overrideHash uint32) (string, error) { - content := fmt.Sprintf( - "%s%d%d%d", - org, - exp.Year(), - exp.Month(), - exp.Day(), - ) - var checksum uint32 - if overrideHash != 0 { - checksum = overrideHash - } else { - checksum = crc32.ChecksumIEEE([]byte(content)) - } - lk := &config.LicenseKey{ - Organization: org, - ExpirationYear: uint16(exp.Year()), - ExpirationMonth: uint8(exp.Month()), - ExpirationDay: uint8(exp.Day()), - Checksum: checksum, - } - bs, err := json.Marshal(lk) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(bs), nil -} - -func TestCheckLicenseKey(t *testing.T) { - msg := "Please get a new one at https://vectorized.io/download-trial/" - tests := []struct { - name string - key func() (string, error) - expected string - }{ - { - name: "key is valid if the hashes match and the expiration date hasn't passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "", - }, - { - name: "key is invalid if the hashes don't match", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 123) - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if the expiration date has already passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(-48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "Your license key has expired. " + msg, - }, - { - name: "key is invalid if it's not base64-encoded", - key: func() (string, error) { - return "this is defs not base 64", nil - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if it's not valid base64-encoded JSON", - key: func() (string, error) { - jsonStr := []byte("{\"thisJson\":\"is invalid\"") - return base64.StdEncoding.EncodeToString(jsonStr), nil - }, - expected: "Invalid license key. " + msg, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(st *testing.T) { - key, err := tt.key() - require.NoError(st, err) - err = config.CheckLicenseKey(key) - if tt.expected == "" { - require.Nil(st, err) - return - } - require.EqualError(st, err, tt.expected) - }) - } -} diff --git a/src/go/rpk/pkg/config/manager.go b/src/go/rpk/pkg/config/manager.go deleted file mode 100644 index a8a098b1fc718..0000000000000 --- a/src/go/rpk/pkg/config/manager.go +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - - "github.com/google/uuid" - "github.com/mitchellh/mapstructure" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" -) - -type Manager interface { - // Reads the config from the given path - Read(path string) (*Config, error) - // Writes the config to Config.ConfigFile - Write(conf *Config) error - // Writes the currently-loaded config to redpanda.config_file - WriteLoaded() error - // Get the currently-loaded config - Get() (*Config, error) - // Sets key to the given value (parsing it according to the format) - Set(key, value, format string) error - // If path is empty, tries to find the file in the default locations. - // Otherwise, it tries to read the file and load it. If the file doesn't - // exist, it tries to create it with the default configuration. - FindOrGenerate(path string) (*Config, error) - // Tries reading a config file at the given path, or tries to find it in - // the default locations if it doesn't exist. - ReadOrFind(path string) (*Config, error) - // Generates and writes the node's UUID - WriteNodeUUID(conf *Config) error - // Merges an input config to the currently-loaded map - Merge(conf *Config) error -} - -type manager struct { - fs afero.Fs - v *viper.Viper -} - -func NewManager(fs afero.Fs) Manager { - return &manager{fs, InitViper(fs)} -} - -func (m *manager) FindOrGenerate(path string) (*Config, error) { - if path == "" { - addConfigPaths(m.v) - err := m.v.ReadInConfig() - if err == nil { - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - if !notFound { - return nil, err - } - path = Default().ConfigFile - } - return readOrGenerate(m.fs, m.v, path) -} - -// Tries reading a config file at the given path, or generates a default config -// and writes it to the path. -func readOrGenerate(fs afero.Fs, v *viper.Viper, path string) (*Config, error) { - abs, err := absPath(path) - if err != nil { - return nil, err - } - v.SetConfigFile(abs) - err = v.ReadInConfig() - if err == nil { - // The config file's there, there's nothing to do. - return unmarshal(v) - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - notExist := os.IsNotExist(err) - if err != nil && !notFound && !notExist { - return nil, fmt.Errorf( - "An error happened while trying to read %s: %v", - abs, - err, - ) - } - log.Debug(err) - log.Infof( - "Couldn't find config file at %s. Generating it.", - abs, - ) - v.Set("config_file", abs) - err = createConfigDir(fs, abs) - if err != nil { - return nil, err - } - err = v.WriteConfigAs(abs) - if err != nil { - return nil, fmt.Errorf( - "Couldn't write config to %s: %v", - abs, - err, - ) - } - return unmarshal(v) -} - -func (m *manager) ReadOrFind(path string) (*Config, error) { - var err error - if path == "" { - path, err = FindConfigFile(m.fs) - if err != nil { - return nil, err - } - } - return m.Read(path) -} - -func (m *manager) Read(path string) (*Config, error) { - // If the path was set, try reading only from there. - abs, err := filepath.Abs(path) - if err != nil { - return nil, err - } - m.v.SetConfigFile(abs) - err = m.v.ReadInConfig() - if err != nil { - return nil, err - } - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err -} - -func (m *manager) WriteNodeUUID(conf *Config) error { - id, err := uuid.NewUUID() - if err != nil { - return err - } - conf.NodeUUID = id.String() - return m.Write(conf) -} - -func (m *manager) Get() (*Config, error) { - return unmarshal(m.v) -} - -// Checks config and writes it to the given path. -func (m *manager) Write(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - // Merge the config into a new viper.Viper instance to prevent - // concurrent writes to the underlying config map. - v := InitViper(m.fs) - current, err := unmarshal(m.v) - if err != nil { - return err - } - currentMap, err := toMap(current) - if err != nil { - return err - } - v.MergeConfigMap(currentMap) - v.MergeConfigMap(confMap) - return checkAndWrite(m.fs, v, conf.ConfigFile) -} - -// Writes the currently loaded config. -func (m *manager) WriteLoaded() error { - return checkAndWrite(m.fs, m.v, m.v.GetString("config_file")) -} - -func write(fs afero.Fs, v *viper.Viper, path string) error { - err := createConfigDir(fs, path) - if err != nil { - return err - } - err = v.WriteConfigAs(path) - if err != nil { - return err - } - log.Debugf( - "Configuration written to %s.", - path, - ) - return nil -} - -func (m *manager) setDeduceFormat(key, value string) error { - replace := func(key string, newValue interface{}) error { - newV := viper.New() - newV.Set(key, newValue) - return m.v.MergeConfigMap(newV.AllSettings()) - } - - var newVal interface{} - switch { - case json.Unmarshal([]byte(value), &newVal) == nil: // Try JSON - return replace(key, newVal) - - case yaml.Unmarshal([]byte(value), &newVal) == nil: // Try YAML - return replace(key, newVal) - - default: // Treat the value as a "single" - m.v.Set(key, parse(value)) - return nil - } -} - -func (m *manager) Set(key, value, format string) error { - if key == "" { - return errors.New("empty config field key") - } - if format == "" { - return m.setDeduceFormat(key, value) - } - var newConfValue interface{} - switch strings.ToLower(format) { - case "single": - m.v.Set(key, parse(value)) - return nil - case "yaml": - err := yaml.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - case "json": - err := json.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - default: - return fmt.Errorf("unsupported format %s", format) - } - newV := viper.New() - newV.Set(key, newConfValue) - return m.v.MergeConfigMap(newV.AllSettings()) -} - -func (m *manager) Merge(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - return m.v.MergeConfigMap(confMap) -} - -func checkAndWrite(fs afero.Fs, v *viper.Viper, path string) error { - ok, errs := check(v) - if !ok { - reasons := []string{} - for _, err := range errs { - reasons = append(reasons, err.Error()) - } - return errors.New(strings.Join(reasons, ", ")) - } - lastBackupFile, err := findBackup(fs, filepath.Dir(path)) - if err != nil { - return err - } - exists, err := afero.Exists(fs, path) - if err != nil { - return err - } - if !exists { - // If the config doesn't exist, just write it. - return write(fs, v, path) - } - // Otherwise, backup the current config file, write the new one, and - // try to recover if there's an error. - log.Debug("Backing up the current config") - backup, err := utils.BackupFile(fs, path) - if err != nil { - return err - } - log.Debugf("Backed up the current config to %s", backup) - if lastBackupFile != "" && lastBackupFile != backup { - log.Debug("Removing previous backup file") - err = fs.Remove(lastBackupFile) - if err != nil { - return err - } - } - log.Debugf("Writing the new redpanda config to '%s'", path) - err = write(fs, v, path) - if err != nil { - return recover(fs, backup, path, err) //nolint:revive // false positive: this recover function is different from built-in recover - } - return nil -} - -func recover(fs afero.Fs, backup, path string, err error) error { - log.Infof("Recovering the previous confing from %s", backup) - recErr := utils.CopyFile(fs, backup, path) - if recErr != nil { - msg := "couldn't persist the new config due to '%v'," + - " nor recover the backup due to '%v" - return fmt.Errorf(msg, err, recErr) - } - return fmt.Errorf("couldn't persist the new config due to '%v'", err) -} - -func unmarshal(v *viper.Viper) (*Config, error) { - result := &Config{} - decoderConfig := decoderConfig() - decoderConfig.Result = result - decoder, err := mapstructure.NewDecoder(&decoderConfig) - if err != nil { - return nil, err - } - err = decoder.Decode(v.AllSettings()) - if err != nil { - return nil, err - } - if result.ConfigFile == "" { - result.ConfigFile, err = absPath(v.ConfigFileUsed()) - if err != nil { - return nil, err - } - } - return result, nil -} - -// Redpanda version < 21.1.4 only supported a single anonymous listener and a -// single anonymous advertised address. This custom decode function translates -// a single SocketAddress-equivalent map[string]interface{} into a -// []NamedSocketAddress. -func v21_1_4MapToNamedSocketAddressSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]NamedSocketAddress{}) { - if from.Kind() == reflect.Map { - sa := NamedSocketAddress{} - err := mapstructure.Decode(data, &sa) - if err != nil { - return nil, err - } - return []NamedSocketAddress{sa}, nil - } - } - return data, nil -} - -// Redpanda version <= 21.4.1 only supported a single TLS config. This custom -// decode function translates a single TLS config-equivalent -// map[string]interface{} into a []ServerTLS. -//nolint:revive // using underscore here is intended -func v21_4_1TlsMapToNamedTlsSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]ServerTLS{}) { - if from.Kind() == reflect.Map { - tls := ServerTLS{} - err := mapstructure.Decode(data, &tls) - if err != nil { - return nil, err - } - return []ServerTLS{tls}, nil - } - } - return data, nil -} - -func parse(val string) interface{} { - if i, err := strconv.Atoi(val); err == nil { - return i - } - if f, err := strconv.ParseFloat(val, 64); err == nil { - return f - } - if b, err := strconv.ParseBool(val); err == nil { - return b - } - return val -} - -func absPath(path string) (string, error) { - absPath, err := filepath.Abs(path) - if err != nil { - return "", fmt.Errorf( - "couldn't convert the used config file path to"+ - " absolute: %s", - path, - ) - } - return absPath, nil -} - -func createConfigDir(fs afero.Fs, configFile string) error { - dir := filepath.Dir(configFile) - err := fs.MkdirAll(dir, 0o755) - if err != nil { - return fmt.Errorf( - "couldn't create config dir %s: %v", - dir, - err, - ) - } - return nil -} diff --git a/src/go/rpk/pkg/config/params.go b/src/go/rpk/pkg/config/params.go index ed701b43e7763..f1cc87c56ba33 100644 --- a/src/go/rpk/pkg/config/params.go +++ b/src/go/rpk/pkg/config/params.go @@ -10,20 +10,23 @@ package config import ( + "encoding/json" "errors" "fmt" "net" "os" "path/filepath" + "reflect" "strconv" "strings" "syscall" "time" + "github.com/google/uuid" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // This file contains the Params type, which will eventually be created in @@ -255,6 +258,9 @@ func (p *Params) Load(fs afero.Fs) (*Config, error) { Rpk: RpkConfig{ CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } if err := p.readConfig(fs, c); err != nil { @@ -552,3 +558,142 @@ func (c *Config) addUnsetDefaults() { r.AdminAPI.Addresses = []string{"127.0.0.1:9644"} } } + +// Set allow to set a single configuration property by passing a key value pair +// +// Key: string containing the yaml property tag, e.g: 'rpk.admin_api'. +// Value: string representation of the value, either single value or partial +// representation if a format is passed (yaml / json). +// Format: either single, json, or yaml (default: single). +func (c *Config) Set(key, value, format string) error { + if key == "" { + return fmt.Errorf("key field must not be empty") + } + props := strings.Split(key, ".") + rv := reflect.ValueOf(c).Elem() + found, other, err := getField(props, rv) + if err != nil { + return err + } + isOther := other != reflect.Value{} + + field := found + if isOther { + field = other + } + + if field.CanAddr() { + i := field.Addr().Interface() + in := value + switch strings.ToLower(format) { + case "yaml", "single", "": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("%s: %s", p, value) + } + err = yaml.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + case "json": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("{%q: %q}", p, value) + } + err = json.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + default: + return fmt.Errorf("unsupported format %s", format) + } + } + return fmt.Errorf("this is a bug... please fill an issue") +} + +// getField deeply search in p for the value that reflect property props. +func getField(props []string, p reflect.Value) (reflect.Value, reflect.Value, error) { + if len(props) == 0 { + return p, reflect.Value{}, nil + } + if p.Kind() == reflect.Slice && p.Len() > 0 { + p = p.Index(0) + } + if p.Kind() == reflect.Slice && p.Len() == 0 { + p.Set(reflect.New(p.Type().Elem())) + } + // If is a nil pointer we assign the zero value, and we reassign p to the + // value that p points to + if p.Kind() == reflect.Ptr { + if p.IsNil() { + p.Set(reflect.New(p.Type().Elem())) + } + p = reflect.Indirect(p) + } + if p.Kind() == reflect.Struct { + newP, other, err := getFieldByTag(props[0], p) + if err != nil { + return reflect.Value{}, reflect.Value{}, err + } + // if is "Other" map field, we stop the recursion and return + if (other != reflect.Value{}) { + // user may try to set deep unmanaged property: + // rpk.unmanaged.name = "name" + if len(props) > 1 { + return reflect.Value{}, reflect.Value{}, fmt.Errorf("cannot set property %v", strings.Join(props, ".")) + } + return reflect.Value{}, other, nil + } + return getField(props[1:], newP) + } + return reflect.Value{}, reflect.Value{}, fmt.Errorf("cannot recurse on type %v", p.Type()) +} + +// getFieldByTag finds a field with a given yaml tag and returns 3 parameters: +// +// 1. if tag is found within the struct, return the field. +// 2. if tag is not found _but_ the struct has "Other" field, return Other. +// 3. Error if it can't find the given tag and "Other" field is unavailable. +func getFieldByTag(tag string, p reflect.Value) (reflect.Value, reflect.Value, error) { + t := p.Type() + var other bool + // Loop struct to get the field that match tag. + for i := 0; i < p.NumField(); i++ { + // Rpk blindly set configuration parameters, we parse these parameters + // in the "Other" field. + if t.Field(i).Name == "Other" { + other = true + } + yt := t.Field(i).Tag.Get("yaml") + + // yaml struct tags can contain flags such as omitempty, + // when tag.Get("yaml") is called it will return + // "my_tag,omitempty" + // so we only need first parameter of the string slice. + ft := strings.Split(yt, ",")[0] + + if ft == tag { + return p.Field(i), reflect.Value{}, nil + } else { + continue + } + } + + // If we can't find the tag but the struct has an 'Other' map field: + if other { + return reflect.Value{}, p.FieldByName("Other"), nil + } + + return reflect.Value{}, reflect.Value{}, fmt.Errorf("couldn't find property %q", tag) +} + +func (c *Config) WriteNodeUUID() error { + id, err := uuid.NewUUID() + if err != nil { + return err + } + c.NodeUUID = id.String() + return nil +} diff --git a/src/go/rpk/pkg/config/params_test.go b/src/go/rpk/pkg/config/params_test.go index e8528bc7883f3..bda974d6682b2 100644 --- a/src/go/rpk/pkg/config/params_test.go +++ b/src/go/rpk/pkg/config/params_test.go @@ -1,11 +1,12 @@ package config import ( + "fmt" "strings" "testing" "github.com/spf13/afero" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func TestParams_Write(t *testing.T) { @@ -20,27 +21,27 @@ func TestParams_Write(t *testing.T) { name: "create default config file if there is no config file yet", exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: /var/lib/redpanda/data - node_id: 0 - seed_servers: [] - rpc_server: - address: 0.0.0.0 - port: 33145 - kafka_api: - - address: 0.0.0.0 - port: 9092 - admin: - - address: 0.0.0.0 - port: 9644 - developer_mode: true + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - kafka_api: - brokers: - - 0.0.0.0:9092 - admin_api: - addresses: - - 127.0.0.1:9644 - enable_usage_stats: false + kafka_api: + brokers: + - 0.0.0.0:9092 + admin_api: + addresses: + - 127.0.0.1:9644 + enable_usage_stats: false `, }, { @@ -49,8 +50,8 @@ rpk: cfgToWrite: &Config{ConfigFile: "/etc/redpanda/redpanda.yaml", Redpanda: RedpandaConfig{ID: 6}}, exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: "" - node_id: 6 + data_directory: "" + node_id: 6 `, }, } @@ -100,6 +101,7 @@ redpanda: t.Errorf("unexpected error while reading the file in %s", path) return } + fmt.Println(string(b)) if !strings.Contains(string(b), test.exp) { t.Errorf("string:\n%v, does not contain expected:\n%v", string(b), test.exp) return diff --git a/src/go/rpk/pkg/config/schema.go b/src/go/rpk/pkg/config/schema.go index fb5f9aa629152..3dd6727cd19a5 100644 --- a/src/go/rpk/pkg/config/schema.go +++ b/src/go/rpk/pkg/config/schema.go @@ -87,7 +87,8 @@ type KafkaClient struct { } type SeedServer struct { - Host SocketAddress `yaml:"host" mapstructure:"host" json:"host"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` } type SocketAddress struct { diff --git a/src/go/rpk/pkg/config/weak.go b/src/go/rpk/pkg/config/weak.go index 96e97d45b9ad6..012a7dc8cb515 100644 --- a/src/go/rpk/pkg/config/weak.go +++ b/src/go/rpk/pkg/config/weak.go @@ -246,7 +246,7 @@ func (s *serverTLSArray) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) + fmt.Fprintf(os.Stderr, "redpanda.yaml: Server TLS must be an array; a non-array value in array fields is deprecated and will be removed in the future\n") // do not log serverTLS value here *s = []ServerTLS{single} return nil } @@ -333,6 +333,7 @@ func (rpc *RedpandaConfig) UnmarshalYAML(n *yaml.Node) error { DeveloperMode weakBool `yaml:"developer_mode"` Other map[string]interface{} `yaml:",inline"` } + if err := n.Decode(&internal); err != nil { return err } @@ -533,6 +534,30 @@ func (s *ServerTLS) UnmarshalYAML(n *yaml.Node) error { return nil } +func (ss *SeedServer) UnmarshalYAML(n *yaml.Node) error { + var internal struct { + Address weakString `yaml:"address"` + Port weakInt `yaml:"port"` + // deprecated + Host *SocketAddress `yaml:"host"` + NodeID *weakInt `yaml:"node_id"` + } + if err := n.Decode(&internal); err != nil { + return err + } + if internal.NodeID != nil { + fmt.Println("redpanda yaml: redpanda.seed_servers.node_id is deprecated and unused.") + } + if internal.Host != nil { + internal.Address = weakString(internal.Host.Address) + internal.Port = weakInt(internal.Host.Port) + fmt.Println("redpanda yaml: redpanda.seed_servers.host is deprecated and will be removed in the future, use redpanda.seed_servers.{address,port}") + } + ss.Address = string(internal.Address) + ss.Port = int(internal.Port) + return nil +} + func (sa *SocketAddress) UnmarshalYAML(n *yaml.Node) error { var internal struct { Address weakString `yaml:"address"` diff --git a/src/go/rpk/pkg/config/weak_test.go b/src/go/rpk/pkg/config/weak_test.go index 3f578868a8635..62ca4f08b0942 100644 --- a/src/go/rpk/pkg/config/weak_test.go +++ b/src/go/rpk/pkg/config/weak_test.go @@ -545,7 +545,7 @@ func TestSeedServers(t *testing.T) { port: 80 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, + {"0.0.0.1", 80}, }, }, { @@ -559,8 +559,8 @@ func TestSeedServers(t *testing.T) { port: 90 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, - {Host: SocketAddress{"0.0.0.2", 90}}, + {"0.0.0.1", 80}, + {"0.0.0.2", 90}, }, }, { @@ -647,9 +647,8 @@ redpanda: name: external port: 9093 seed_servers: - - host: - address: 192.168.0.1 - port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -761,7 +760,7 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {"192.168.0.1", 33145}, }, Other: map[string]interface{}{ "enable_admin_api": true, @@ -876,9 +875,15 @@ redpanda: name: external port: 9093 seed_servers: + - host: + address: 192.168.0.1 + port: 33145 + - node_id: "0" host: address: 192.168.0.1 port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -989,7 +994,9 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {"192.168.0.1", 33145}, + {"192.168.0.1", 33145}, + {"192.168.0.1", 33145}, }, Other: map[string]interface{}{ "enable_admin_api": true, diff --git a/src/go/rpk/pkg/plugin/manifest.go b/src/go/rpk/pkg/plugin/manifest.go index e631ccbb07fdb..ecca50478f13b 100644 --- a/src/go/rpk/pkg/plugin/manifest.go +++ b/src/go/rpk/pkg/plugin/manifest.go @@ -15,7 +15,7 @@ import ( "strings" "time" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Manifest represents a plugin manifest, which is essentially a list of diff --git a/src/go/rpk/pkg/tuners/iotune/data.go b/src/go/rpk/pkg/tuners/iotune/data.go index e0af774aa69c1..3c1b631eab18c 100644 --- a/src/go/rpk/pkg/tuners/iotune/data.go +++ b/src/go/rpk/pkg/tuners/iotune/data.go @@ -14,7 +14,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cloud/vendor" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type IoProperties struct { diff --git a/src/go/rpk/pkg/yaml/yaml.go b/src/go/rpk/pkg/yaml/yaml.go deleted file mode 100644 index 47385a64312cb..0000000000000 --- a/src/go/rpk/pkg/yaml/yaml.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package yaml - -import ( - "github.com/spf13/afero" - "gopkg.in/yaml.v2" -) - -func Persist(fs afero.Fs, in interface{}, file string) error { - bytes, err := yaml.Marshal(in) - if err != nil { - return err - } - err = afero.WriteFile(fs, file, bytes, 0o644) - return err -} - -func Read(fs afero.Fs, out interface{}, file string) error { - content, err := afero.ReadFile(fs, file) - if err != nil { - return err - } - err = yaml.Unmarshal(content, out) - if err != nil { - return err - } - return nil -} diff --git a/tests/rptest/tests/rpk_config_test.py b/tests/rptest/tests/rpk_config_test.py index 04ab14608629d..56ef2d0d2e7a8 100644 --- a/tests/rptest/tests/rpk_config_test.py +++ b/tests/rptest/tests/rpk_config_test.py @@ -39,37 +39,37 @@ def test_config_init(self): # node_uuid: (the uuid is random so we don't compare it) pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false + enable_usage_stats: false + tune_network: false + tune_disk_scheduler: false + tune_disk_nomerges: false + tune_disk_write_cache: false + tune_disk_irq: false + tune_fstrim: false + tune_cpu: false + tune_aio_events: false + tune_clocksource: false + tune_swappiness: false + tune_transparent_hugepages: false + enable_memory_locking: false + tune_coredump: false + coredump_dir: /var/lib/redpanda/coredump + tune_ballast_file: false + overprovisioned: false schema_registry: {} ''' @@ -151,19 +151,19 @@ def test_config_set_json(self): rpk.config_set(key, value, format='json') expected_config = yaml.full_load(''' -coredump_dir: /var/lib/redpanda/coredump -enable_memory_locking: false -enable_usage_stats: false -tune_aio_events: true -tune_clocksource: false -tune_coredump: false -tune_cpu: true -tune_disk_irq: true -tune_disk_nomerges: false -tune_disk_scheduler: false -tune_fstrim: false -tune_network: false -tune_swappiness: false + enable_usage_stats: false + tune_network: false + tune_disk_scheduler: false + tune_disk_nomerges: false + tune_disk_irq: true + tune_fstrim: false + tune_cpu: true + tune_aio_events: true + tune_clocksource: false + tune_swappiness: false + enable_memory_locking: false + tune_coredump: false + coredump_dir: /var/lib/redpanda/coredump ''') with tempfile.TemporaryDirectory() as d: