diff --git a/src/go/k8s/cmd/configurator/main.go b/src/go/k8s/cmd/configurator/main.go index b019540579ec4..d83e003dc520b 100644 --- a/src/go/k8s/cmd/configurator/main.go +++ b/src/go/k8s/cmd/configurator/main.go @@ -99,8 +99,8 @@ func main() { log.Print(c.String()) fs := afero.NewOsFs() - mgr := config.NewManager(fs) - cfg, err := mgr.Read(path.Join(c.configSourceDir, "redpanda.yaml")) + p := config.Params{ConfigPath: path.Join(c.configSourceDir, "redpanda.yaml")} + cfg, err := p.Load(fs) if err != nil { log.Fatalf("%s", fmt.Errorf("unable to read the redpanda configuration file: %w", err)) } diff --git a/src/go/k8s/go.mod b/src/go/k8s/go.mod index 84024a5ce8978..e1724e93623c8 100644 --- a/src/go/k8s/go.mod +++ b/src/go/k8s/go.mod @@ -15,7 +15,7 @@ require ( github.com/redpanda-data/redpanda/src/go/rpk v0.0.0-00010101000000-000000000000 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.21.4 k8s.io/apimachinery v0.21.4 k8s.io/client-go v0.21.4 @@ -48,16 +48,12 @@ require ( github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -65,12 +61,8 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/sethgrid/pester v1.1.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect - github.com/spf13/cast v1.3.0 // indirect github.com/spf13/cobra v1.1.3 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/twmb/tlscfg v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -86,7 +78,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.52.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.21.4 // indirect diff --git a/src/go/k8s/go.sum b/src/go/k8s/go.sum index 5701a2bb3447f..19114ba780fbf 100644 --- a/src/go/k8s/go.sum +++ b/src/go/k8s/go.sum @@ -58,7 +58,6 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= @@ -324,7 +323,6 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -364,7 +362,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -400,7 +397,6 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -430,7 +426,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -467,8 +462,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= @@ -516,7 +509,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -577,9 +569,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -587,7 +577,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -595,7 +584,6 @@ github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -604,7 +592,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -618,7 +605,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -1001,7 +987,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1023,8 +1008,9 @@ gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/k8s/pkg/resources/configmap.go b/src/go/k8s/pkg/resources/configmap.go index d747ec8d8c424..83472292472ad 100644 --- a/src/go/k8s/pkg/resources/configmap.go +++ b/src/go/k8s/pkg/resources/configmap.go @@ -333,11 +333,9 @@ func (r *ConfigMapResource) CreateConfiguration( replicas := *r.pandaCluster.Spec.Replicas for i := int32(0); i < replicas; i++ { cr.SeedServers = append(cr.SeedServers, config.SeedServer{ - Host: config.SocketAddress{ - // Example address: cluster-sample-0.cluster-sample.default.svc.cluster.local - Address: fmt.Sprintf("%s-%d.%s", r.pandaCluster.Name, i, r.serviceFQDN), - Port: clusterCRPortOrRPKDefault(c.RPCServer.Port, cr.RPCServer.Port), - }, + // Example address: cluster-sample-0.cluster-sample.default.svc.cluster.local + Address: fmt.Sprintf("%s-%d.%s", r.pandaCluster.Name, i, r.serviceFQDN), + Port: clusterCRPortOrRPKDefault(c.RPCServer.Port, cr.RPCServer.Port), }) } diff --git a/src/go/k8s/pkg/resources/configmap_test.go b/src/go/k8s/pkg/resources/configmap_test.go index 7a2be09fe0e2d..350c73b783bb2 100644 --- a/src/go/k8s/pkg/resources/configmap_test.go +++ b/src/go/k8s/pkg/resources/configmap_test.go @@ -104,7 +104,7 @@ func TestEnsureConfigMap_AdditionalConfig(t *testing.T) { name: "Primitive object in additional configuration", additionalConfiguration: map[string]string{"redpanda.transactional_id_expiration_ms": "25920000000"}, expectedStrings: []string{"transactional_id_expiration_ms: 25920000000"}, - expectedHash: "0cb36f0be0d64032a61eb51a5d2985ea", + expectedHash: "9df8ec83985d2c4f5584ea15fdac892b", }, { name: "Complex struct in additional configuration", @@ -114,7 +114,7 @@ func TestEnsureConfigMap_AdditionalConfig(t *testing.T) { - address: 0.0.0.0 port: 8081 name: external`}, - expectedHash: "4697714fe9b8f8bcaebb814b93f2b8f6", + expectedHash: "8d4152f81e60ac6973f511bc30e24961", }, { name: "shadow index cache directory", @@ -122,7 +122,7 @@ func TestEnsureConfigMap_AdditionalConfig(t *testing.T) { `cloud_storage_cache_directory: /var/lib/shadow-index-cache`, `cloud_storage_cache_size: "10737418240"`, }, - expectedHash: "2f51e71fa4b673fb105f98cb09cb7a00", + expectedHash: "f101931ee6ef54d62c164ff64937eba5", }, } for _, tc := range testcases { diff --git a/src/go/k8s/pkg/resources/configuration/configuration_modes.go b/src/go/k8s/pkg/resources/configuration/configuration_modes.go index 30d46e2898a55..22ac7a42fd456 100644 --- a/src/go/k8s/pkg/resources/configuration/configuration_modes.go +++ b/src/go/k8s/pkg/resources/configuration/configuration_modes.go @@ -14,9 +14,6 @@ import ( "reflect" "strconv" "strings" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/spf13/afero" ) // GlobalConfigurationMode changes the behavior of the global configuration when reading/writing properties. @@ -65,33 +62,20 @@ func (r globalConfigurationModeClassic) GetAdditionalRedpandaProperty( func (r globalConfigurationModeClassic) SetAdditionalFlatProperties( targetConfig *GlobalConfiguration, props map[string]string, ) error { - // all properties are node properties in the classic setting - mgr := config.NewManager(afero.NewOsFs()) - err := mgr.Merge(&targetConfig.NodeConfiguration) - if err != nil { - return fmt.Errorf("merging node configuration: %w", err) - } - // Add arbitrary parameters to configuration for k, v := range props { if builtInType(v) { - err = mgr.Set(k, v, "single") + err := targetConfig.NodeConfiguration.Set(k, v, "single") if err != nil { return fmt.Errorf("setting built-in type: %w", err) } } else { - err = mgr.Set(k, v, "") + err := targetConfig.NodeConfiguration.Set(k, v, "") if err != nil { return fmt.Errorf("setting complex type: %w", err) } } } - - newRpCfg, err := mgr.Get() - if err != nil { - return fmt.Errorf("getting redpanda node configuration: %w", err) - } - targetConfig.NodeConfiguration = *newRpCfg return nil } diff --git a/src/go/rpk/go.mod b/src/go/rpk/go.mod index 454c82a4709a7..9fcaaf55d4daf 100644 --- a/src/go/rpk/go.mod +++ b/src/go/rpk/go.mod @@ -17,7 +17,6 @@ require ( github.com/google/uuid v1.1.1 github.com/hashicorp/go-multierror v1.1.0 github.com/lorenzosaino/go-sysctl v0.1.0 - github.com/mitchellh/mapstructure v1.4.1 github.com/olekukonko/tablewriter v0.0.1 github.com/opencontainers/image-spec v1.0.1 github.com/pkg/errors v0.9.1 @@ -29,7 +28,6 @@ require ( github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.7.0 github.com/tklauser/go-sysconf v0.1.0 github.com/twmb/franz-go v1.6.0 @@ -40,8 +38,7 @@ require ( golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -49,20 +46,17 @@ require ( github.com/containerd/containerd v1.4.8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/godbus/dbus/v5 v5.0.3 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/protobuf v1.4.2 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect @@ -71,12 +65,8 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tklauser/numcpus v0.1.0 // indirect golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect @@ -86,6 +76,5 @@ require ( google.golang.org/grpc v1.27.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.51.0 // indirect gotest.tools/v3 v3.0.3 // indirect ) diff --git a/src/go/rpk/go.sum b/src/go/rpk/go.sum index f96d58a2c356a..b83b8a7b83d4a 100644 --- a/src/go/rpk/go.sum +++ b/src/go/rpk/go.sum @@ -16,7 +16,6 @@ github.com/AlecAivazis/survey/v2 v2.3.2 h1:TqTB+aDDCLYhf9/bD2TwSO8u8jDSmMUd2SUVO github.com/AlecAivazis/survey/v2 v2.3.2/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= @@ -77,8 +76,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -123,7 +120,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -149,7 +145,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -164,7 +159,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -190,7 +184,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lorenzosaino/go-sysctl v0.1.0 h1:BfWlLYErjQeCb0TB3kzIq5nsVVjKqyT0NKvWqifz7gE= github.com/lorenzosaino/go-sysctl v0.1.0/go.mod h1:jp4+NUTRTq8/3QPxrhPPav8j/tCw1yUwZtG0iPWBFcU= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -216,8 +209,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -233,7 +224,6 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -275,9 +265,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= @@ -285,16 +273,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -304,7 +289,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tklauser/go-sysconf v0.1.0 h1:f4fOH/B3HZYy2wz7o9Vr5i4dBhOKcedooHq7/4c0hxQ= github.com/tklauser/go-sysconf v0.1.0/go.mod h1:h54uFIrVIJBr8RXt3F5JJdxVkmFeallWuXajbMhn2O8= @@ -401,7 +385,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -490,18 +473,16 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go index e2bc13650ceb9..53d5b2e833196 100644 --- a/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go +++ b/src/go/rpk/pkg/cli/cmd/cluster/config/reset.go @@ -16,7 +16,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func newForceResetCommand(fs afero.Fs) *cobra.Command { diff --git a/src/go/rpk/pkg/cli/cmd/config.go b/src/go/rpk/pkg/cli/cmd/config.go index 6061658cce7e1..1051630ed50b2 100644 --- a/src/go/rpk/pkg/cli/cmd/config.go +++ b/src/go/rpk/pkg/cli/cmd/config.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewConfigCommand(fs, mgr), + redpanda.NewConfigCommand(fs), "rpk redpanda config", ) } diff --git a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go index f4c900d636bd9..4653cd321b272 100644 --- a/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go +++ b/src/go/rpk/pkg/cli/cmd/debug/bundle_linux.go @@ -43,7 +43,7 @@ import ( "github.com/spf13/afero" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func executeBundle( diff --git a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go index 5d9a32f2c48e0..8d78a55234039 100644 --- a/src/go/rpk/pkg/cli/cmd/generate/prometheus.go +++ b/src/go/rpk/pkg/cli/cmd/generate/prometheus.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/cobra" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type ScrapeConfig struct { diff --git a/src/go/rpk/pkg/cli/cmd/mode.go b/src/go/rpk/pkg/cli/cmd/mode.go index a300c74111a24..802d8b1b1b242 100644 --- a/src/go/rpk/pkg/cli/cmd/mode.go +++ b/src/go/rpk/pkg/cli/cmd/mode.go @@ -15,13 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewModeCommand(mgr), + redpanda.NewModeCommand(fs), "rpk redpanda mode", ) } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda.go b/src/go/rpk/pkg/cli/cmd/redpanda.go index 14d68a75320e4..e0c980a5cd3af 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda.go @@ -15,26 +15,23 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda/admin" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewRedpandaCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewRedpandaCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { command := &cobra.Command{ Use: "redpanda", Short: "Interact with a local Redpanda process", } - command.AddCommand(redpanda.NewStartCommand(fs, mgr, launcher)) - command.AddCommand(redpanda.NewStopCommand(fs, mgr)) + command.AddCommand(redpanda.NewStartCommand(fs, launcher)) + command.AddCommand(redpanda.NewStopCommand(fs)) command.AddCommand(redpanda.NewCheckCommand(fs)) command.AddCommand(redpanda.NewTuneCommand(fs)) - command.AddCommand(redpanda.NewModeCommand(mgr)) - command.AddCommand(redpanda.NewConfigCommand(fs, mgr)) + command.AddCommand(redpanda.NewModeCommand(fs)) + command.AddCommand(redpanda.NewConfigCommand(fs)) command.AddCommand(admin.NewCommand(fs)) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config.go b/src/go/rpk/pkg/cli/cmd/redpanda/config.go index b79a8d0f857a6..b82b8d5df9011 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config.go @@ -17,54 +17,47 @@ import ( "fmt" "net" + "github.com/google/uuid" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" vnet "github.com/redpanda-data/redpanda/src/go/rpk/pkg/net" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/spf13/afero" "github.com/spf13/cobra" ) const configFileFlag = "config" -func NewConfigCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewConfigCommand(fs afero.Fs) *cobra.Command { root := &cobra.Command{ Use: "config ", Short: "Edit configuration.", } - root.AddCommand(set(fs, mgr)) - root.AddCommand(bootstrap(mgr)) - root.AddCommand(initNode(mgr)) + root.AddCommand(set(fs)) + root.AddCommand(bootstrap(fs)) + root.AddCommand(initNode(fs)) return root } -func set(fs afero.Fs, mgr config.Manager) *cobra.Command { +func set(fs afero.Fs) *cobra.Command { var ( format string configPath string ) c := &cobra.Command{ Use: "set ", - Short: "Set configuration values, such as the node IDs or the list of seed servers.", + Short: "Set configuration values, such as the node IDs or the list of seed servers", Args: cobra.ExactArgs(2), - RunE: func(_ *cobra.Command, args []string) error { - var err error - key := args[0] - value := args[1] - if configPath == "" { - configPath, err = config.FindConfigFile(fs) - if err != nil { - return err - } - } - _, err = mgr.Read(configPath) - if err != nil { - return err - } - err = mgr.Set(key, value, format) - if err != nil { - return err - } - return mgr.WriteLoaded() + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + + err = cfg.Set(args[0], args[1], format) + out.MaybeDieErr(err) + + err = cfg.Write(fs) + out.MaybeDieErr(err) }, } c.Flags().StringVar(&format, @@ -84,7 +77,7 @@ func set(fs afero.Fs, mgr config.Manager) *cobra.Command { return c } -func bootstrap(mgr config.Manager) *cobra.Command { +func bootstrap(fs afero.Fs) *cobra.Command { var ( ips []string self string @@ -93,44 +86,36 @@ func bootstrap(mgr config.Manager) *cobra.Command { ) c := &cobra.Command{ Use: "bootstrap --id [--self ] [--ips ]", - Short: "Initialize the configuration to bootstrap a cluster.", + Short: "Initialize the configuration to bootstrap a cluster", Long: helpBootstrap, - Args: cobra.OnlyValidArgs, - RunE: func(c *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + seeds, err := parseSeedIPs(ips) - if err != nil { - return err - } - var ownIP net.IP - if self != "" { - ownIP = net.ParseIP(self) - if ownIP == nil { - return fmt.Errorf("%s is not a valid IP", self) - } - } else { - ownIP, err = getOwnIP() - if err != nil { - return err - } - } - conf.Redpanda.ID = id - conf.Redpanda.RPCServer.Address = ownIP.String() - conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ + out.MaybeDieErr(err) + + ownIP, err := parseSelfIP(self) + out.MaybeDieErr(err) + + cfg.Redpanda.ID = id + cfg.Redpanda.RPCServer.Address = ownIP.String() + cfg.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultKafkaPort, }} - conf.Redpanda.AdminAPI = []config.NamedSocketAddress{{ + cfg.Redpanda.AdminAPI = []config.NamedSocketAddress{{ Address: ownIP.String(), Port: config.DefaultAdminPort, }} - conf.Redpanda.SeedServers = []config.SeedServer{} - conf.Redpanda.SeedServers = seeds - return mgr.Write(conf) + cfg.Redpanda.SeedServers = []config.SeedServer{} + cfg.Redpanda.SeedServers = seeds + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringSliceVar( @@ -162,22 +147,27 @@ func bootstrap(mgr config.Manager) *cobra.Command { return c } -func initNode(mgr config.Manager) *cobra.Command { +func initNode(fs afero.Fs) *cobra.Command { var configPath string c := &cobra.Command{ Use: "init", - Short: "Init the node after install, by setting the node's UUID.", - Args: cobra.OnlyValidArgs, - RunE: func(_ *cobra.Command, args []string) error { - conf, err := mgr.FindOrGenerate(configPath) - if err != nil { - return err - } + Short: "Init the node after install, by setting the node's UUID", + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(args) + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + // Don't reset the node's UUID if it has already been set. - if conf.NodeUUID == "" { - return mgr.WriteNodeUUID(conf) + if cfg.NodeUUID == "" { + id, err := uuid.NewUUID() + out.MaybeDie(err, "error creating nodeUUID: %v", err) + cfg.NodeUUID = id.String() } - return nil + + err = cfg.Write(fs) + out.MaybeDie(err, "error writing config file: %v", err) }, } c.Flags().StringVar( @@ -190,6 +180,22 @@ func initNode(mgr config.Manager) *cobra.Command { return c } +func parseSelfIP(self string) (net.IP, error) { + if self != "" { + ownIP := net.ParseIP(self) + if ownIP == nil { + return nil, fmt.Errorf("%s is not a valid IP", self) + } + return ownIP, nil + } else { + ownIP, err := getOwnIP() + if err != nil { + return nil, err + } + return ownIP, nil + } +} + func parseSeedIPs(ips []string) ([]config.SeedServer, error) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port var seeds []config.SeedServer @@ -202,10 +208,8 @@ func parseSeedIPs(ips []string) ([]config.SeedServer, error) { host, port := vnet.SplitHostPortDefault(hostport, defaultRPCPort) seed := config.SeedServer{ - Host: config.SocketAddress{ - Address: host, - Port: port, - }, + Address: host, + Port: port, } seeds = append(seeds, seed) } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go index b69c0e536ac5b..943ab999998ee 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/config_test.go @@ -13,166 +13,15 @@ package redpanda import ( - "path/filepath" "strings" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) -func TestSetCmd(t *testing.T) { - tests := []struct { - name string - key string - value string - args []string - expected interface{} - expectErr bool - }{ - { - name: "it should set single integer fields", - key: "redpanda.node_id", - value: "54312", - expected: 54312, - }, - { - name: "it should set single float fields", - key: "redpanda.float_field", - value: "42.3", - expected: 42.3, - }, - { - name: "it should set single string fields", - key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", - expected: "'/var/lib/differentdir'", - }, - { - name: "it should set single bool fields", - key: "rpk.enable_usage_stats", - value: "true", - expected: true, - }, - { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, - args: []string{"--format", "yaml"}, - expected: map[string]interface{}{ - "enable_usage_stats": false, - "overprovisioned": false, - "tune_network": false, - "tune_disk_scheduler": false, - "tune_disk_write_cache": false, - "tune_disk_nomerges": false, - "tune_disk_irq": true, - "tune_cpu": false, - "tune_aio_events": false, - "tune_clocksource": false, - "tune_swappiness": false, - "tune_transparent_hugepages": false, - "enable_memory_locking": false, - "tune_fstrim": false, - "tune_coredump": false, - "tune_ballast_file": false, - "coredump_dir": "/var/lib/redpanda/coredump", - }, - }, - { - name: "it should partially set map fields (json)", - key: "redpanda.kafka_api", - value: `[{ - "address": "192.168.54.2", - "port": 9092 -}]`, - args: []string{"--format", "json"}, - expected: []interface{}{ - map[interface{}]interface{}{ - "port": 9092, - "address": "192.168.54.2", - }, - }, - }, - { - name: "it should fail if the new value is invalid", - key: "redpanda", - value: `{"data_directory": ""}`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (json)", - key: "redpanda", - value: `{"seed_servers": []`, - args: []string{"--format", "json"}, - expectErr: true, - }, - { - name: "it should fail if the value isn't well formatted (yaml)", - key: "redpanda", - value: `seed_servers: -- host: - address: "123.`, - args: []string{"--format", "yaml"}, - expectErr: true, - }, - { - name: "it should fail if the format isn't supported", - key: "redpanda", - value: `node_id=1`, - args: []string{"--format", "toml"}, - expectErr: true, - }, - { - name: "it should fail if no key is passed", - value: `node_id=1`, - expectErr: true, - }, - { - name: "it should fail if no value is passed", - key: "rpk.tune_coredump", - expectErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - - c := NewConfigCommand(fs, mgr) - args := []string{"set"} - if tt.key != "" { - args = append(args, tt.key) - } - if tt.value != "" { - args = append(args, tt.value) - } - c.SetArgs(append(args, tt.args...)) - err = c.Execute() - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) - val := v.Get(tt.key) - require.Exactly(t, tt.expected, val) - }) - } -} - func TestBootstrap(t *testing.T) { defaultRPCPort := config.Default().Redpanda.RPCServer.Port tests := []struct { @@ -193,22 +42,16 @@ func TestBootstrap(t *testing.T) { ips: []string{"187.89.76.3", "192.168.34.5", "192.168.45.8"}, expSeedServers: []config.SeedServer{ { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: defaultRPCPort, - }, + Address: "187.89.76.3", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "192.168.34.5", - Port: defaultRPCPort, - }, + Address: "192.168.34.5", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "192.168.45.8", - Port: defaultRPCPort, - }, + Address: "192.168.45.8", + Port: defaultRPCPort, }, }, self: "192.168.34.5", @@ -219,72 +62,36 @@ func TestBootstrap(t *testing.T) { ips: []string{"187.89.76.3", "localhost", "redpanda.com", "test-url.net", "187.89.76.3:80"}, expSeedServers: []config.SeedServer{ { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: defaultRPCPort, - }, + Address: "187.89.76.3", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "localhost", - Port: defaultRPCPort, - }, + Address: "localhost", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "redpanda.com", - Port: defaultRPCPort, - }, + Address: "redpanda.com", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "test-url.net", - Port: defaultRPCPort, - }, + Address: "test-url.net", + Port: defaultRPCPort, }, { - Host: config.SocketAddress{ - Address: "187.89.76.3", - Port: 80, - }, + Address: "187.89.76.3", + Port: 80, }, }, self: "192.168.34.5", id: "1", }, - { - name: "it should fail if any of the --ips IPs isn't valid", - ips: []string{"187.89.9", "192.168.34.5", "192.168.45.8"}, - self: "192.168.34.5", - id: "1", - expectedErr: `invalid host "187.89.9" does not match "host", nor "host:port", nor "scheme://host:port"`, - }, - { - name: "it should fail if --self isn't a valid IP", - ips: []string{"187.89.9.78", "192.168.34.5", "192.168.45.8"}, - self: "www.host.com", - id: "1", - expectedErr: "www.host.com is not a valid IP", - }, - { - name: "it should fail if --id isn't passed", - expectedErr: "required flag(s) \"id\" not set", - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - configPath, err := filepath.Abs("./redpanda.yaml") - require.NoError(t, err) fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - err = fs.MkdirAll( - filepath.Dir(configPath), - 0o644, - ) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"bootstrap", "--config", configPath} + c := bootstrap(fs) + var args []string if len(tt.ips) != 0 { args = append( args, @@ -299,15 +106,15 @@ func TestBootstrap(t *testing.T) { args = append(args, "--id", tt.id) } c.SetArgs(args) - err = c.Execute() + err := c.Execute() if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return } require.NoError(t, err) - _, err = fs.Stat(configPath) + _, err = fs.Stat(config.Default().ConfigFile) require.NoError(t, err) - conf, err := mgr.Read(configPath) + conf, err := new(config.Params).Load(fs) require.NoError(t, err) require.Equal(t, tt.self, conf.Redpanda.RPCServer.Address) @@ -327,25 +134,37 @@ func TestBootstrap(t *testing.T) { } func TestInitNode(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - conf := config.Default() - err := mgr.Write(conf) - require.NoError(t, err) - c := NewConfigCommand(fs, mgr) - args := []string{"init"} - c.SetArgs(args) + for _, test := range []struct { + name string + prevID string + }{ + {name: "without UUID"}, + {name: "with UUID", prevID: "my_id"}, + } { + t.Run(test.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + c := config.Default() + if test.prevID != "" { + c.NodeUUID = test.prevID + } - err = c.Execute() - require.NoError(t, err) + bs, err := yaml.Marshal(c) + require.NoError(t, err) + err = afero.WriteFile(fs, c.ConfigFile, bs, 0o644) + require.NoError(t, err) + + cmd := initNode(fs) + err = cmd.Execute() + require.NoError(t, err) - v := viper.New() - v.SetFs(fs) - v.SetConfigType("yaml") - v.SetConfigFile(conf.ConfigFile) - err = v.ReadInConfig() - require.NoError(t, err) + conf, err := new(config.Params).Load(fs) + require.NoError(t, err) - val := v.Get("node_uuid") - require.NotEmpty(t, val) + if test.prevID != "" { + require.Exactly(t, conf.NodeUUID, test.prevID) + } else { + require.NotEmpty(t, conf.NodeUUID) + } + }) + } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go index 0378d76fd86f1..fc88a5dfee543 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode.go @@ -17,11 +17,12 @@ import ( "strings" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - log "github.com/sirupsen/logrus" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" + "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewModeCommand(mgr config.Manager) *cobra.Command { +func NewModeCommand(fs afero.Fs) *cobra.Command { var configFile string command := &cobra.Command{ Use: "mode ", @@ -33,9 +34,9 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { } return nil }, - RunE: func(_ *cobra.Command, args []string) error { - // Safe to access args[0] because it was validated in Args - return executeMode(mgr, configFile, args[0]) + Run: func(cmd *cobra.Command, args []string) { + err := executeMode(fs, cmd, args) + out.MaybeDieErr(err) }, } command.Flags().StringVar( @@ -48,15 +49,23 @@ func NewModeCommand(mgr config.Manager) *cobra.Command { return command } -func executeMode(mgr config.Manager, configFile string, mode string) error { - conf, err := mgr.FindOrGenerate(configFile) +func executeMode(fs afero.Fs, cmd *cobra.Command, args []string) error { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + if err != nil { + return fmt.Errorf("unable to load config: %v", err) + } + // Safe to access args[0] because it was validated in Args + mode := args[0] + cfg, err = config.SetMode(mode, cfg) if err != nil { return err } - conf, err = config.SetMode(mode, conf) + + fmt.Printf("Writing '%s' mode defaults to '%s'", mode, cfg.ConfigFile) + err = cfg.Write(fs) if err != nil { return err } - log.Infof("Writing '%s' mode defaults to '%s'", mode, conf.ConfigFile) - return mgr.Write(conf) + return nil } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go index b65f4be4d1b21..a3a5c437a507e 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/mode_test.go @@ -13,17 +13,14 @@ package redpanda import ( - "bytes" - "fmt" "os" - "strings" + "path/filepath" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func fillRpkConfig(path, mode string) *config.Config { @@ -45,23 +42,21 @@ func fillRpkConfig(path, mode string) *config.Config { Overprovisioned: !val, TuneBallastFile: val, } + // Unset defaults that get added after command execution, needed to compare + // expected config with loaded config. + conf.Rpk.KafkaAPI = config.RpkKafkaAPI{Brokers: []string{"0.0.0.0:9092"}} + conf.Rpk.AdminAPI = config.RpkAdminAPI{Addresses: []string{"127.0.0.1:9644"}} return conf } func TestModeCommand(t *testing.T) { configPath := "/etc/redpanda/redpanda.yaml" - dir, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - wdConfigPath := fmt.Sprintf("%s/redpanda.yaml", dir) tests := []struct { name string args []string before func(afero.Fs) (string, error) expectedConfig *config.Config - expectedOutput string - expectedErrMsg string + expectedError bool }{ { name: "development mode should disable all fields in the rpk config", @@ -74,8 +69,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'development' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "production mode should enable all fields in the rpk config", @@ -88,8 +81,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'production' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "the development mode alias, 'dev', should work the same", @@ -102,8 +93,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeDev), - expectedOutput: fmt.Sprintf("Writing 'dev' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "the production mode alias, 'prod', should work the same", @@ -116,8 +105,6 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "mode should work if --config isn't passed, but the file is in /etc/redpanda/redpanda.yaml", @@ -130,46 +117,43 @@ func TestModeCommand(t *testing.T) { return configPath, afero.WriteFile(fs, configPath, bs, 0o644) }, expectedConfig: fillRpkConfig(configPath, config.ModeProd), - expectedOutput: fmt.Sprintf("Writing 'prod' mode defaults to '%s'", configPath), - expectedErrMsg: "", }, { name: "mode lists the available modes if the one passed is not valid", args: []string{"invalidmode"}, before: func(fs afero.Fs) (string, error) { + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + wdConfigPath := filepath.Join(dir, "/redpanda.yaml") bs, err := yaml.Marshal(fillRpkConfig(wdConfigPath, config.ModeDev)) if err != nil { return "", err } return wdConfigPath, afero.WriteFile(fs, wdConfigPath, bs, 0o644) }, - expectedOutput: "", - expectedErrMsg: "'invalidmode' is not a supported mode. Available modes: dev, development, prod, production", + expectedError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) - path, err := tt.before(fs) + _, err := tt.before(fs) require.NoError(t, err) - var out bytes.Buffer - cmd := NewModeCommand(mgr) + cmd := NewModeCommand(fs) cmd.SetArgs(tt.args) - logrus.SetOutput(&out) - err = cmd.Execute() - if tt.expectedErrMsg != "" { - require.EqualError(t, err, tt.expectedErrMsg) + err = executeMode(fs, cmd, tt.args) + if tt.expectedError && err != nil { return } require.NoError(t, err) - output := out.String() - require.Contains(t, strings.TrimSpace(output), tt.expectedOutput) - mgr = config.NewManager(fs) - conf, err := mgr.Read(path) + + conf, err := new(config.Params).Load(fs) require.NoError(t, err) - require.Exactly(t, tt.expectedConfig, conf) + + require.Exactly(t, tt.expectedConfig, conf.File()) }) } } diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start.go b/src/go/rpk/pkg/cli/cmd/redpanda/start.go index b362bfad50908..5b063901710d8 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start.go @@ -113,9 +113,7 @@ func parseConfigKvs(args []string) ([]string, []string) { return kvs, args } -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { prestartCfg := prestartConfig{} var ( configFile string @@ -143,25 +141,25 @@ func NewStartCommand( // (POSIX standard) UnknownFlags: true, }, - RunE: func(ccmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { // --set flags have to be parsed by hand because pflag (the // underlying flag-parsing lib used by cobra) uses a CSV parser // for list flags, and since JSON often contains commas, it // blows up when there's a JSON object. configKvs, filteredArgs := parseConfigKvs(os.Args) - conf, err := mgr.FindOrGenerate(configFile) + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) if err != nil { - return err + return fmt.Errorf("unable to load config file: %s", err) } if len(configKvs) > 0 { - conf, err = setConfig(mgr, configKvs) - if err != nil { + if err = setConfig(cfg, configKvs); err != nil { return err } } - updateConfigWithFlags(conf, ccmd.Flags()) + updateConfigWithFlags(cfg, cmd.Flags()) env := api.EnvironmentPayload{} if len(seeds) == 0 { @@ -177,11 +175,11 @@ func NewStartCommand( } seedServers, err := parseSeeds(seeds) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(seedServers) != 0 { - conf.Redpanda.SeedServers = seedServers + cfg.Redpanda.SeedServers = seedServers } kafkaAddr = stringSliceOr( @@ -196,11 +194,11 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(kafkaAPI) > 0 { - conf.Redpanda.KafkaAPI = kafkaAPI + cfg.Redpanda.KafkaAPI = kafkaAPI } proxyAddr = stringSliceOr( @@ -215,14 +213,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(proxyAPI) > 0 { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.PandaproxyAPI = proxyAPI + cfg.Pandaproxy.PandaproxyAPI = proxyAPI } schemaRegAddr = stringSliceOr( @@ -237,14 +235,14 @@ func NewStartCommand( config.DefaultSchemaRegPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if len(schemaRegAPI) > 0 { - if conf.SchemaRegistry == nil { - conf.SchemaRegistry = config.Default().SchemaRegistry + if cfg.SchemaRegistry == nil { + cfg.SchemaRegistry = config.Default().SchemaRegistry } - conf.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI + cfg.SchemaRegistry.SchemaRegistryAPI = schemaRegAPI } rpcAddr = stringOr( @@ -256,11 +254,11 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if rpcServer != nil { - conf.Redpanda.RPCServer = *rpcServer + cfg.Redpanda.RPCServer = *rpcServer } advertisedKafka = stringSliceOr( @@ -275,11 +273,12 @@ func NewStartCommand( config.DefaultKafkaPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - if advKafkaAPI != nil { - conf.Redpanda.AdvertisedKafkaAPI = advKafkaAPI + + if len(advKafkaAPI) > 0 { + cfg.Redpanda.AdvertisedKafkaAPI = advKafkaAPI } advertisedProxy = stringSliceOr( @@ -294,14 +293,14 @@ func NewStartCommand( config.DefaultProxyPort, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advProxyAPI != nil { - if conf.Pandaproxy == nil { - conf.Pandaproxy = config.Default().Pandaproxy + if cfg.Pandaproxy == nil { + cfg.Pandaproxy = config.Default().Pandaproxy } - conf.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI + cfg.Pandaproxy.AdvertisedPandaproxyAPI = advProxyAPI } advertisedRPC = stringOr( @@ -313,53 +312,53 @@ func NewStartCommand( config.Default().Redpanda.RPCServer.Port, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } if advRPCApi != nil { - conf.Redpanda.AdvertisedRPCAPI = advRPCApi + cfg.Redpanda.AdvertisedRPCAPI = advRPCApi } installDirectory, err := cli.GetOrFindInstallDir(fs, installDirFlag) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } rpArgs, err := buildRedpandaFlags( fs, - conf, + cfg, filteredArgs, sFlags, - ccmd.Flags(), + cmd.Flags(), !prestartCfg.checkEnabled, ) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } checkPayloads, tunerPayloads, err := prestart( fs, rpArgs, - conf, + cfg, prestartCfg, timeout, ) env.Checks = checkPayloads env.Tuners = tunerPayloads if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - err = mgr.Write(conf) + err = cfg.Write(fs) if err != nil { - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, err) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, err) return err } - sendEnv(fs, env, conf, !prestartCfg.checkEnabled, nil) + sendEnv(fs, env, cfg, !prestartCfg.checkEnabled, nil) rpArgs.ExtraArgs = args - log.Info(common.FeedbackMsg) - log.Info("Starting redpanda...") + fmt.Println(common.FeedbackMsg) + fmt.Println("Starting redpanda...") return launcher.Start(installDirectory, rpArgs) }, } @@ -640,21 +639,21 @@ func mergeFlags( return current } -func setConfig(mgr config.Manager, configKvs []string) (*config.Config, error) { +func setConfig(cfg *config.Config, configKvs []string) error { for _, rawKv := range configKvs { parts := strings.SplitN(rawKv, "=", 2) if len(parts) < 2 { - return nil, fmt.Errorf( + return fmt.Errorf( "key-value pair '%s' is not formatted as expected (k=v)", rawKv, ) } - err := mgr.Set(parts[0], parts[1], "") + err := cfg.Set(parts[0], parts[1], "") if err != nil { - return nil, err + return err } } - return mgr.Get() + return nil } func resolveWellKnownIo( @@ -873,7 +872,7 @@ func parseSeeds(seeds []string) ([]config.SeedServer, error) { } seedServers = append( seedServers, - config.SeedServer{Host: *addr}, + config.SeedServer{Address: addr.Address, Port: addr.Port}, ) } return seedServers, nil diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go index 5fb8afc148759..6786e0302f50f 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/start_test.go @@ -14,7 +14,9 @@ package redpanda import ( "bytes" + "net" "os" + "strconv" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" @@ -107,18 +109,10 @@ func TestParseSeeds(t *testing.T) { name: "it should parse well-formed seed addrs", arg: []string{"127.0.0.1:1234", "domain.com:9892", "lonely-host", "192.168.34.1"}, expected: []config.SeedServer{ - { - Host: config.SocketAddress{Address: "127.0.0.1", Port: 1234}, - }, - { - Host: config.SocketAddress{Address: "domain.com", Port: 9892}, - }, - { - Host: config.SocketAddress{Address: "lonely-host", Port: 33145}, - }, - { - Host: config.SocketAddress{Address: "192.168.34.1", Port: 33145}, - }, + {Address: "127.0.0.1", Port: 1234}, + {Address: "domain.com", Port: 9892}, + {Address: "lonely-host", Port: 33145}, + {Address: "192.168.34.1", Port: 33145}, }, }, { @@ -171,7 +165,7 @@ func TestStartCommand(t *testing.T) { 0o755, ) }, - expectedErrMsg: "An error happened while trying to read /etc/redpanda/redpanda.yaml: While parsing config: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml` into map[string]interface {}", + expectedErrMsg: "unable to load config file: unable to yaml decode /etc/redpanda/redpanda.yaml: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `^¬yaml`", }, { name: "should generate the config at the given path if it doesn't exist", args: []string{ @@ -191,11 +185,15 @@ func TestStartCommand(t *testing.T) { "The config should have been created at '%s'", path, ) - defaultConf := config.Default() - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + c := config.Default() + // Adding unset default that get added on first load. + b0 := c.Redpanda.KafkaAPI[0] + c.Rpk.KafkaAPI.Brokers = []string{net.JoinHostPort(b0.Address, strconv.Itoa(b0.Port))} + c.Rpk.AdminAPI.Addresses = []string{"127.0.0.1:9644"} + + conf, err := new(config.Params).Load(fs) require.NoError(st, err) - require.Exactly(st, defaultConf, conf) + require.Exactly(st, c, conf.File()) }, }, { name: "it should write the given config file path", @@ -208,8 +206,8 @@ func TestStartCommand(t *testing.T) { }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} // In command execution this will be done by with ParamsFromCommand + conf, err := p.Load(fs) require.NoError(st, err) require.Exactly(st, path, conf.ConfigFile) }, @@ -255,9 +253,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -321,9 +318,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) expectedAdmin := []config.NamedSocketAddress{{ Address: "192.168.54.2", @@ -375,9 +371,8 @@ func TestStartCommand(t *testing.T) { } }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := testConfigPath - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + p := &config.Params{ConfigPath: "/arbitrary/path/redpanda.yaml"} + conf, err := p.Load(fs) require.NoError(st, err) // The value set through the --kafka-addr flag should // have been picked. @@ -396,9 +391,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -408,13 +401,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) - conf := config.Default() - return mgr.Write(conf) + cfg := config.Default() + return cfg.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, config.Default().ConfigFile, conf.ConfigFile) }, @@ -426,9 +417,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, 34, conf.Redpanda.ID) }, @@ -439,9 +428,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, config.Default().Redpanda.ID, conf.Redpanda.ID) @@ -452,14 +439,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.ID = 98 - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -475,9 +460,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly(st, "aws:i3xlarge:default", conf.Rpk.WellKnownIo) }, @@ -488,14 +471,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.WellKnownIo = "gcp:n2standard:ssd" - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) require.Exactly( st, @@ -513,9 +494,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, false, conf.Rpk.Overprovisioned) @@ -526,13 +505,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -549,9 +526,7 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - path := config.Default().ConfigFile - mgr := config.NewManager(fs) - conf, err := mgr.Read(path) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly(st, true, conf.Rpk.EnableMemoryLocking) @@ -563,14 +538,12 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.EnableMemoryLocking = true - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) // Check that the generated config is as expected. require.Exactly( @@ -586,24 +559,17 @@ func TestStartCommand(t *testing.T) { "--seeds", "192.168.34.32:33145,somehost:54321,justahostnoport", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "192.168.34.32", - Port: 33145, - }, + Address: "192.168.34.32", + Port: 33145, }, { - Host: config.SocketAddress{ - Address: "somehost", - Port: 54321, - }, + Address: "somehost", + Port: 54321, }, { - Host: config.SocketAddress{ - Address: "justahostnoport", - Port: 33145, - }, + Address: "justahostnoport", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -620,24 +586,17 @@ func TestStartCommand(t *testing.T) { "-s", "192.168.123.32:33146,host", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "192.168.3.32", - Port: 33145, - }, + Address: "192.168.3.32", + Port: 33145, }, { - Host: config.SocketAddress{ - Address: "192.168.123.32", - Port: 33146, - }, + Address: "192.168.123.32", + Port: 33146, }, { - Host: config.SocketAddress{ - Address: "host", - Port: 33145, - }, + Address: "host", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -659,19 +618,14 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_SEEDS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }, { - Host: config.SocketAddress{ - Address: "host", - Port: 33145, - }, + Address: "host", + Port: 33145, }} // Check that the generated config is as expected. require.Exactly( @@ -686,25 +640,19 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.SeedServers = []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedSeeds := []config.SeedServer{{ - Host: config.SocketAddress{ - Address: "10.23.12.5", - Port: 33146, - }, + Address: "10.23.12.5", + Port: 33146, }} // Check that the generated config is as expected. require.Exactly( @@ -732,8 +680,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -753,8 +700,7 @@ func TestStartCommand(t *testing.T) { "--rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.34.32", @@ -787,8 +733,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "host", @@ -807,17 +752,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.RPCServer = config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := config.SocketAddress{ Address: "192.168.33.33", @@ -837,8 +780,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -858,8 +800,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -879,8 +820,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -901,8 +841,7 @@ func TestStartCommand(t *testing.T) { "--kafka-addr", "nondefaultname://192.168.34.32,host:9092", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Name: "nondefaultname", @@ -939,8 +878,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -959,17 +897,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.KafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -989,8 +925,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1010,8 +945,7 @@ func TestStartCommand(t *testing.T) { "--advertise-kafka-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1044,8 +978,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_KAFKA_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1064,17 +997,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedKafkaAPI = []config.NamedSocketAddress{{ Address: "192.168.33.33", Port: 9892, }} - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.33.33", @@ -1094,8 +1025,7 @@ func TestStartCommand(t *testing.T) { "--advertise-pandaproxy-addr", "192.168.34.32:8083", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "192.168.34.32", @@ -1121,8 +1051,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_PANDAPROXY_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := []config.NamedSocketAddress{{ Address: "host", @@ -1142,8 +1071,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32:33145", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1163,8 +1091,7 @@ func TestStartCommand(t *testing.T) { "--advertise-rpc-addr", "192.168.34.32", }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.34.32", @@ -1197,8 +1124,7 @@ func TestStartCommand(t *testing.T) { os.Unsetenv("REDPANDA_ADVERTISE_RPC_ADDRESS") }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "host", @@ -1217,17 +1143,15 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Redpanda.AdvertisedRPCAPI = &config.SocketAddress{ Address: "192.168.33.33", Port: 9892, } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) { - mgr := config.NewManager(fs) - conf, err := mgr.Read(config.Default().ConfigFile) + conf, err := new(config.Params).Load(fs) require.NoError(st, err) expectedAddr := &config.SocketAddress{ Address: "192.168.33.33", @@ -1246,10 +1170,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--overprovisioned", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--overprovisioned"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--overprovisioned' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--overprovisioned' directly to `rpk start`.", }, { @@ -1258,10 +1181,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--smp", "1", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--smp=1"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--smp' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--smp' directly to `rpk start`.", }, { @@ -1270,10 +1192,9 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", "--memory", "2G", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{"--memory=1G"} - return mgr.Write(conf) + return conf.Write(fs) }, expectedErrMsg: "Configuration conflict. Flag '--memory' is also present in 'rpk.additional_start_flags' in configuration file '/etc/redpanda/redpanda.yaml'. Please remove it and pass '--memory' directly to `rpk start`.", }, { @@ -1282,12 +1203,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--smp=3", "--smp=55", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1302,12 +1222,11 @@ func TestStartCommand(t *testing.T) { "--install-dir", "/var/lib/redpanda", }, before: func(fs afero.Fs) error { - mgr := config.NewManager(fs) conf := config.Default() conf.Rpk.AdditionalStartFlags = []string{ "--logger-log-level=archival=debug:cloud_storage=debug", } - return mgr.Write(conf) + return conf.Write(fs) }, postCheck: func( _ afero.Fs, @@ -1362,7 +1281,6 @@ func TestStartCommand(t *testing.T) { defer tt.after() } fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) var launcher redpanda.Launcher = &noopLauncher{} if tt.launcher != nil { launcher = tt.launcher @@ -1372,11 +1290,11 @@ func TestStartCommand(t *testing.T) { } var out bytes.Buffer logrus.SetOutput(&out) - c := NewStartCommand(fs, mgr, launcher) + c := NewStartCommand(fs, launcher) c.SetArgs(tt.args) err := c.Execute() if tt.expectedErrMsg != "" { - require.EqualError(st, err, tt.expectedErrMsg) + require.Contains(st, err.Error(), tt.expectedErrMsg) return } require.NoError(st, err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go index 5f452ba823732..837b3f6177e28 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop.go @@ -20,13 +20,14 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/os" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/out" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" log "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { var ( configFile string timeout time.Duration @@ -38,8 +39,13 @@ func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { first sends SIGINT, and waits for the specified timeout. Then, if redpanda hasn't stopped, it sends SIGTERM. Lastly, it sends SIGKILL if it's still running.`, - RunE: func(ccmd *cobra.Command, args []string) error { - return executeStop(fs, mgr, configFile, timeout) + Run: func(cmd *cobra.Command, args []string) { + p := config.ParamsFromCommand(cmd) + cfg, err := p.Load(fs) + out.MaybeDie(err, "unable to load config: %v", err) + + err = executeStop(fs, cfg, timeout) + out.MaybeDieErr(err) }, } command.Flags().StringVar( @@ -63,14 +69,8 @@ running.`, return command } -func executeStop( - fs afero.Fs, mgr config.Manager, configFile string, timeout time.Duration, -) error { - conf, err := mgr.ReadOrFind(configFile) - if err != nil { - return err - } - pidFile := conf.PIDFile() +func executeStop(fs afero.Fs, cfg *config.Config, timeout time.Duration) error { + pidFile := cfg.PIDFile() isLocked, err := os.CheckLocked(pidFile) if err != nil { log.Debugf("error checking if the PID file is locked: %v", err) diff --git a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go index f6e4319ba4f38..7b7cc1a48be74 100644 --- a/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go +++ b/src/go/rpk/pkg/cli/cmd/redpanda/stop_test.go @@ -55,7 +55,6 @@ func TestStopCommand(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := config.NewManager(fs) conf := config.Default() command := baseCommand // trap the signals we want to ignore, to check that the @@ -76,11 +75,11 @@ func TestStopCommand(t *testing.T) { conf.PIDFile(), ) require.NoError(t, err) - err = mgr.Write(conf) + err = conf.Write(fs) require.NoError(t, err) var out bytes.Buffer - c := cmd.NewStopCommand(fs, mgr) + c := cmd.NewStopCommand(fs) args := append([]string{"--config", conf.ConfigFile}, tt.args...) c.SetArgs(args) diff --git a/src/go/rpk/pkg/cli/cmd/root.go b/src/go/rpk/pkg/cli/cmd/root.go index c8f83f450d518..947ed95fe0598 100644 --- a/src/go/rpk/pkg/cli/cmd/root.go +++ b/src/go/rpk/pkg/cli/cmd/root.go @@ -37,7 +37,6 @@ import ( func Execute() { verbose := false fs := afero.NewOsFs() - mgr := config.NewManager(fs) if !term.IsTerminal(int(os.Stdout.Fd())) { color.NoColor = true @@ -77,7 +76,7 @@ func Execute() { plugincmd.NewCommand(fs), ) - addPlatformDependentCmds(fs, mgr, root) + addPlatformDependentCmds(fs, root) // To support autocompletion even for plugins, we list all plugins now // and add tiny commands to our root command. Cobra works by creating diff --git a/src/go/rpk/pkg/cli/cmd/root_darwin.go b/src/go/rpk/pkg/cli/cmd/root_darwin.go index c2ad57b703d78..c16d212d62781 100644 --- a/src/go/rpk/pkg/cli/cmd/root_darwin.go +++ b/src/go/rpk/pkg/cli/cmd/root_darwin.go @@ -10,13 +10,10 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { cmd.AddCommand(NewRedpandaDarwinCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/root_linux.go b/src/go/rpk/pkg/cli/cmd/root_linux.go index c71ca1b26a9d7..8b66245fddf49 100644 --- a/src/go/rpk/pkg/cli/cmd/root_linux.go +++ b/src/go/rpk/pkg/cli/cmd/root_linux.go @@ -10,23 +10,20 @@ package cmd import ( - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func addPlatformDependentCmds( - fs afero.Fs, mgr config.Manager, cmd *cobra.Command, -) { - cmd.AddCommand(NewRedpandaCommand(fs, mgr, redpanda.NewLauncher())) +func addPlatformDependentCmds(fs afero.Fs, cmd *cobra.Command) { + cmd.AddCommand(NewRedpandaCommand(fs, redpanda.NewLauncher())) cmd.AddCommand(NewTuneCommand(fs)) cmd.AddCommand(NewCheckCommand(fs)) cmd.AddCommand(NewIoTuneCmd(fs)) - cmd.AddCommand(NewStartCommand(fs, mgr, redpanda.NewLauncher())) - cmd.AddCommand(NewStopCommand(fs, mgr)) - cmd.AddCommand(NewConfigCommand(fs, mgr)) + cmd.AddCommand(NewStartCommand(fs, redpanda.NewLauncher())) + cmd.AddCommand(NewStopCommand(fs)) + cmd.AddCommand(NewConfigCommand(fs)) cmd.AddCommand(NewStatusCommand(fs)) - cmd.AddCommand(NewModeCommand(mgr)) + cmd.AddCommand(NewModeCommand(fs)) } diff --git a/src/go/rpk/pkg/cli/cmd/start.go b/src/go/rpk/pkg/cli/cmd/start.go index db83e7f97b6d4..dfe0251863aeb 100644 --- a/src/go/rpk/pkg/cli/cmd/start.go +++ b/src/go/rpk/pkg/cli/cmd/start.go @@ -15,17 +15,14 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" rp "github.com/redpanda-data/redpanda/src/go/rpk/pkg/redpanda" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStartCommand( - fs afero.Fs, mgr config.Manager, launcher rp.Launcher, -) *cobra.Command { +func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command { return common.Deprecated( - redpanda.NewStartCommand(fs, mgr, launcher), + redpanda.NewStartCommand(fs, launcher), "rpk redpanda start", ) } diff --git a/src/go/rpk/pkg/cli/cmd/stop.go b/src/go/rpk/pkg/cli/cmd/stop.go index 86adf9d278e08..7485c774adf0c 100644 --- a/src/go/rpk/pkg/cli/cmd/stop.go +++ b/src/go/rpk/pkg/cli/cmd/stop.go @@ -15,14 +15,13 @@ package cmd import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/common" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/redpanda" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" "github.com/spf13/afero" "github.com/spf13/cobra" ) -func NewStopCommand(fs afero.Fs, mgr config.Manager) *cobra.Command { +func NewStopCommand(fs afero.Fs) *cobra.Command { return common.Deprecated( - redpanda.NewStopCommand(fs, mgr), + redpanda.NewStopCommand(fs), "rpk redpanda stop", ) } diff --git a/src/go/rpk/pkg/config/config.go b/src/go/rpk/pkg/config/config.go index 5279fd1ea8c1f..0d08dbe23398f 100644 --- a/src/go/rpk/pkg/config/config.go +++ b/src/go/rpk/pkg/config/config.go @@ -10,16 +10,8 @@ package config import ( - "errors" "fmt" - fp "path/filepath" "strings" - - "github.com/mitchellh/mapstructure" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" ) const ( @@ -35,107 +27,35 @@ const ( DefaultBallastFileSize = "1GiB" ) -func InitViper(fs afero.Fs) *viper.Viper { - v := viper.New() - v.SetFs(fs) - v.SetConfigName("redpanda") - v.SetConfigType("yaml") - - // Viper does not take into account our explicit SetConfigType when - // calling ReadInConfig, instead it internally uses SupportedExts. - // Since we only ever want to load yaml, setting this global disables - // ReadInConfig from using any existing json files. - viper.SupportedExts = []string{"yaml"} - - setDefaults(v) - return v -} - -func addConfigPaths(v *viper.Viper) { - v.AddConfigPath("$HOME") - v.AddConfigPath(fp.Join("etc", "redpanda")) - v.AddConfigPath(".") -} - -func setDefaults(v *viper.Viper) { - var traverse func(tree map[string]interface{}, path ...string) - traverse = func(tree map[string]interface{}, path ...string) { - for key, val := range tree { - if subtree, ok := val.(map[string]interface{}); ok { - traverse(subtree, append(path, key)...) - } else { - v.SetDefault( - strings.Join(append(path, key), "."), - val, - ) - } - } - } - traverse(defaultMap()) -} - func Default() *Config { - conf := &Config{} - err := mapstructure.Decode(defaultMap(), conf) - if err != nil { - panic(err) - } - return conf -} - -func defaultMap() map[string]interface{} { - var defaultListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9092, - } - defaultListeners := []interface{}{defaultListener} - var defaultAdminListener interface{} = map[string]interface{}{ - "address": "0.0.0.0", - "port": 9644, - } - defaultAdminListeners := []interface{}{defaultAdminListener} - return map[string]interface{}{ - "config_file": "/etc/redpanda/redpanda.yaml", - "pandaproxy": Pandaproxy{}, - "schema_registry": SchemaRegistry{}, - "redpanda": map[string]interface{}{ - "data_directory": "/var/lib/redpanda/data", - "rpc_server": map[string]interface{}{ - "address": "0.0.0.0", - "port": 33145, + return &Config{ + ConfigFile: "/etc/redpanda/redpanda.yaml", + Redpanda: RedpandaConfig{ + Directory: "/var/lib/redpanda/data", + RPCServer: SocketAddress{ + Address: "0.0.0.0", + Port: 33145, }, - "kafka_api": defaultListeners, - "admin": defaultAdminListeners, - "node_id": 0, - "seed_servers": []interface{}{}, - "developer_mode": true, + KafkaAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9092, + }}, + AdminAPI: []NamedSocketAddress{{ + Address: "0.0.0.0", + Port: 9644, + }}, + SeedServers: []SeedServer{}, + DeveloperMode: true, }, - "rpk": map[string]interface{}{ - "coredump_dir": "/var/lib/redpanda/coredump", + Rpk: RpkConfig{ + CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } } -func findBackup(fs afero.Fs, dir string) (string, error) { - exists, err := afero.Exists(fs, dir) - if err != nil { - return "", err - } - if !exists { - return "", nil - } - files, err := afero.ReadDir(fs, dir) - if err != nil { - return "", err - } - for _, f := range files { - if strings.HasSuffix(f.Name(), ".bk") { - return fmt.Sprintf("%s/%s", dir, f.Name()), nil - } - } - return "", nil -} - func SetMode(mode string, conf *Config) (*Config, error) { m, err := NormalizeMode(mode) if err != nil { @@ -223,131 +143,71 @@ func AvailableModes() []string { } } -func Check(conf *Config) (bool, []error) { - configMap, err := toMap(conf) - if err != nil { - return false, []error{err} - } - - v := viper.New() - err = v.MergeConfigMap(configMap) - if err != nil { - return false, []error{err} - } - return check(v) -} - -func check(v *viper.Viper) (bool, []error) { - errs := checkRedpandaConfig(v) +func (c *Config) Check() (bool, []error) { + errs := checkRedpandaConfig(c) errs = append( errs, - checkRpkConfig(v)..., + checkRpkConfig(c)..., ) ok := len(errs) == 0 return ok, errs } -func checkRedpandaConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetString("redpanda.data_directory") == "" { +func checkRedpandaConfig(cfg *Config) []error { + var errs []error + rp := cfg.Redpanda + // top level check + if rp.Directory == "" { errs = append(errs, fmt.Errorf("redpanda.data_directory can't be empty")) } - if v.GetInt("redpanda.node_id") < 0 { + if rp.ID < 0 { errs = append(errs, fmt.Errorf("redpanda.node_id can't be a negative integer")) } - rpcServerKey := "redpanda.rpc_server" - exists := v.Sub(rpcServerKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", rpcServerKey), - ) + // rpc server + if rp.RPCServer == (SocketAddress{}) { + errs = append(errs, fmt.Errorf("redpanda.rpc_server missing")) } else { - socket := &SocketAddress{} - err := unmarshalKey(v, rpcServerKey, socket) - if err != nil { - errs = append( - errs, - fmt.Errorf("invalid structure for %s", rpcServerKey), - ) - } else { - errs = append( - errs, - checkSocketAddress(*socket, rpcServerKey)..., - ) + saErrs := checkSocketAddress(rp.RPCServer, "redpanda.rpc_server") + if len(saErrs) > 0 { + errs = append(errs, saErrs...) } } - kafkaAPIKey := "redpanda.kafka_api" - exists = v.Get(kafkaAPIKey) != nil - if !exists { - errs = append( - errs, - fmt.Errorf("%s missing", kafkaAPIKey), - ) + // kafka api + if len(rp.KafkaAPI) == 0 { + errs = append(errs, fmt.Errorf("redpanda.kafka_api missing")) } else { - var kafkaListeners []NamedSocketAddress - err := unmarshalKey(v, "redpanda.kafka_api", &kafkaListeners) - if err != nil { - log.Error(err) - err = fmt.Errorf( - "%s doesn't have the expected structure", - kafkaAPIKey, - ) - return append( - errs, - err, - ) - } - for i, addr := range kafkaListeners { - configPath := fmt.Sprintf( - "%s.%d", - kafkaAPIKey, - i, - ) - errs = append( - errs, - checkSocketAddress( - SocketAddress{addr.Address, addr.Port}, - configPath, - )..., - ) + for i, addr := range rp.KafkaAPI { + configPath := fmt.Sprintf("redpanda.kafka_api.%d", i) + saErrs := checkSocketAddress(SocketAddress{addr.Address, addr.Port}, configPath) + if len(saErrs) > 0 { + errs = append(errs, saErrs...) + } } } - var seedServersSlice []*SeedServer // map[string]interface{} - err := unmarshalKey(v, "redpanda.seed_servers", &seedServersSlice) - if err != nil { - log.Error(err) - msg := "redpanda.seed_servers doesn't have the expected structure" - return append( - errs, - errors.New(msg), - ) - } - if len(seedServersSlice) > 0 { - seedServersPath := "redpanda.seed_servers" - for i, seed := range seedServersSlice { - configPath := fmt.Sprintf( - "%s.%d.host", - seedServersPath, - i, - ) - errs = append( - errs, - checkSocketAddress( - seed.Host, - configPath, - )..., - ) + // seed servers + if len(rp.SeedServers) > 0 { + for i, seed := range rp.SeedServers { + configPath := fmt.Sprintf("redpanda.seed_servers.%d.host", i) + saErrs := checkSocketAddress(SocketAddress(seed), configPath) + errs = append(errs, saErrs...) } } return errs } +func checkRpkConfig(cfg *Config) []error { + var errs []error + if cfg.Rpk.TuneCoredump && cfg.Rpk.CoredumpDir == "" { + errs = append(errs, fmt.Errorf("if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty")) + } + return errs +} + func checkSocketAddress(s SocketAddress, configPath string) []error { - errs := []error{} + var errs []error if s.Port == 0 { errs = append(errs, fmt.Errorf("%s.port can't be 0", configPath)) } @@ -356,62 +216,3 @@ func checkSocketAddress(s SocketAddress, configPath string) []error { } return errs } - -func checkRpkConfig(v *viper.Viper) []error { - errs := []error{} - if v.GetBool("rpk.tune_coredump") && v.GetString("rpk.coredump_dir") == "" { - msg := "if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty" - errs = append(errs, errors.New(msg)) - } - return errs -} - -func decoderConfig() mapstructure.DecoderConfig { - return mapstructure.DecoderConfig{ - // Sometimes viper will save int values as strings (i.e. - // through BindPFlag) so we have to allow mapstructure - // to cast them. - WeaklyTypedInput: true, - DecodeHook: mapstructure.ComposeDecodeHookFunc( - // These 2 hooks are viper's default hooks. - // https://github.com/spf13/viper/blob/fb4eafdd9775508c450b90b1b72affeef4a68cf5/viper.go#L1004-L1005 - // They're set here because when decoderConfigOptions' resulting - // viper.DecoderConfigOption is used, viper's hooks are overridden. - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - // This hook translates the pre-21.1.4 configuration format to the - // latest one (see schema.go) - v21_1_4MapToNamedSocketAddressSlice, - // This hook translates the pre-21.4.1 TLS configuration format to the - // latest one (see schema.go) - v21_4_1TlsMapToNamedTlsSlice, - ), - } -} - -func decoderConfigOptions() viper.DecoderConfigOption { - return func(c *mapstructure.DecoderConfig) { - cfg := decoderConfig() - c.DecodeHook = cfg.DecodeHook - c.WeaklyTypedInput = cfg.WeaklyTypedInput - } -} - -func unmarshalKey(v *viper.Viper, key string, val interface{}) error { - return v.UnmarshalKey( - key, - val, - decoderConfigOptions(), - ) -} - -func toMap(conf *Config) (map[string]interface{}, error) { - mapConf := make(map[string]interface{}) - bs, err := yaml.Marshal(conf) - if err != nil { - return nil, err - } - err = yaml.Unmarshal(bs, &mapConf) - return mapConf, err -} diff --git a/src/go/rpk/pkg/config/config_test.go b/src/go/rpk/pkg/config/config_test.go index 6d7d891090bab..b515df6aafb13 100644 --- a/src/go/rpk/pkg/config/config_test.go +++ b/src/go/rpk/pkg/config/config_test.go @@ -10,25 +10,18 @@ package config import ( - "path/filepath" "testing" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - vyaml "github.com/redpanda-data/redpanda/src/go/rpk/pkg/yaml" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func getValidConfig() *Config { conf := Default() conf.Redpanda.SeedServers = []SeedServer{ - { - SocketAddress{"127.0.0.1", 33145}, - }, - { - SocketAddress{"127.0.0.1", 33146}, - }, + {"127.0.0.1", 33145}, + {"127.0.0.1", 33146}, } conf.Redpanda.DeveloperMode = false conf.Rpk = RpkConfig{ @@ -59,94 +52,93 @@ func TestSet(t *testing.T) { key string value string format string - check func(st *testing.T, c *Config, mgr *manager) + check func(st *testing.T, c *Config) expectErr bool }{ { - name: "it should parse '1' as an int and not as bool (true)", + name: "parse '1' as an int and not as bool (true)", key: "redpanda.node_id", value: "1", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 1, c.Redpanda.ID) }, }, { - name: "it should set single integer fields", + name: "set single integer fields", key: "redpanda.node_id", value: "54312", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should detect single integer fields if format isn't passed", + name: "detect single integer fields if format isn't passed", key: "redpanda.node_id", value: "54312", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, 54312, c.Redpanda.ID) }, }, { - name: "it should set single float fields", + name: "set single float fields", key: "redpanda.float_field", value: "42.3", format: "single", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should detect single float fields if format isn't passed", + name: "detect single float fields if format isn't passed", key: "redpanda.float_field", value: "42.3", - check: func(st *testing.T, _ *Config, mgr *manager) { - // require.True(st, ok, "Config map is of the wrong type") - require.Exactly(st, 42.3, mgr.v.Get("redpanda.float_field")) + check: func(st *testing.T, cfg *Config) { + require.Exactly(st, 42.3, cfg.Redpanda.Other["float_field"]) }, }, { - name: "it should set single string fields", + name: "set single string fields", key: "redpanda.data_directory", - value: "'/var/lib/differentdir'", + value: "/var/lib/differentdir", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "'/var/lib/differentdir'", c.Redpanda.Directory) + check: func(st *testing.T, c *Config) { + require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should detect single string fields if format isn't passed", + name: "detect single string fields if format isn't passed", key: "redpanda.data_directory", value: "/var/lib/differentdir", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, "/var/lib/differentdir", c.Redpanda.Directory) }, }, { - name: "it should set single bool fields", + name: "set single bool fields", key: "rpk.enable_usage_stats", value: "true", format: "single", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should detect single bool fields if format isn't passed", + name: "detect single bool fields if format isn't passed", key: "rpk.enable_usage_stats", value: "true", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { require.Exactly(st, true, c.Rpk.EnableUsageStats) }, }, { - name: "it should partially set map fields (yaml)", - key: "rpk", - value: `tune_disk_irq: true`, + name: "partially set map fields (yaml)", + key: "rpk", + value: `tune_disk_irq: true +tune_cpu: true`, format: "yaml", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := RpkConfig{ EnableUsageStats: false, Overprovisioned: false, @@ -154,7 +146,7 @@ func TestSet(t *testing.T) { TuneDiskScheduler: false, TuneNomerges: false, TuneDiskIrq: true, - TuneCPU: false, + TuneCPU: true, TuneAioEvents: false, TuneClocksource: false, TuneSwappiness: false, @@ -163,21 +155,20 @@ func TestSet(t *testing.T) { TuneFstrim: false, TuneCoredump: false, TuneDiskWriteCache: false, - CoredumpDir: "/var/lib/redpanda/coredump", } require.Exactly(st, expected, c.Rpk) }, }, { - name: "it should detect pandaproxy client single field if format isn't passed", + name: "detect pandaproxy client single field if format isn't passed", key: "pandaproxy_client.retries", value: "42", - check: func(st *testing.T, c *Config, mgr *manager) { - require.Exactly(st, 42.0, c.PandaproxyClient.Other["retries"]) + check: func(st *testing.T, c *Config) { + require.Exactly(st, 42, c.PandaproxyClient.Other["retries"]) }, }, { - name: "it should detect yaml-formatted values if format isn't passed", + name: "detect yaml-formatted values if format isn't passed", key: "redpanda.kafka_api", value: `- name: external address: 192.168.73.45 @@ -186,7 +177,7 @@ func TestSet(t *testing.T) { address: 10.21.34.58 port: 9092 `, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Name: "external", Address: "192.168.73.45", @@ -200,14 +191,14 @@ func TestSet(t *testing.T) { }, }, { - name: "it should partially set map fields (json)", + name: "partially set map fields (json)", key: "redpanda.kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, format: "json", - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -216,13 +207,13 @@ func TestSet(t *testing.T) { }, }, { - name: "it should detect json-formatted values if format isn't passed", + name: "detect json-formatted values if format isn't passed", key: "redpanda.advertised_kafka_api", value: `[{ "address": "192.168.54.2", "port": 9092 }]`, - check: func(st *testing.T, c *Config, _ *manager) { + check: func(st *testing.T, c *Config) { expected := []NamedSocketAddress{{ Port: 9092, Address: "192.168.54.2", @@ -231,14 +222,22 @@ func TestSet(t *testing.T) { }, }, { - name: "it should fail if the value isn't well formatted (json)", + name: "set value of a slice", + key: "redpanda.admin.port", + value: "9641", + check: func(st *testing.T, c *Config) { + require.Exactly(st, 9641, c.Redpanda.AdminAPI[0].Port) + }, + }, + { + name: "fail if the value isn't well formatted (json)", key: "redpanda", value: `{"seed_servers": []`, format: "json", expectErr: true, }, { - name: "it should fail if the value isn't well formatted (yaml)", + name: "fail if the value isn't well formatted (yaml)", key: "redpanda", value: `seed_servers: - host: @@ -247,96 +246,38 @@ func TestSet(t *testing.T) { expectErr: true, }, { - name: "it should fail if the format isn't supported", + name: "fail if the format isn't supported", key: "redpanda", value: "node_id=1", format: "toml", expectErr: true, }, { - name: "it should fail if no key is passed", + name: "fail if no key is passed", value: `node_id=1`, expectErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set(tt.key, tt.value, tt.format) - if tt.expectErr { - require.Error(t, err) - return - } - require.NoError(t, err) - if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) - } - }) - } -} - -func TestMerge(t *testing.T) { - tests := []struct { - name string - config Config - check func(st *testing.T, c *Config, mgr *manager) - expectErr bool - }{ - { - name: "it should merge Kafka API spec", - config: Config{ - Redpanda: RedpandaConfig{ - KafkaAPI: []NamedSocketAddress{{ - Name: "kafka-api-name", - Address: "1.2.3.4", - Port: 9123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "kafka-api-name", c.Redpanda.KafkaAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Redpanda.KafkaAPI[0].Address) - require.Exactly(st, 9123, c.Redpanda.KafkaAPI[0].Port) - }, - }, { - name: "it should merge Pandaproxy API spec", - config: Config{ - Pandaproxy: &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{{ - Name: "proxy-api-name", - Address: "1.2.3.4", - Port: 8123, - }}, - }, - }, - check: func(st *testing.T, c *Config, _ *manager) { - require.Exactly(st, "proxy-api-name", c.Pandaproxy.PandaproxyAPI[0].Name) - require.Exactly(st, "1.2.3.4", c.Pandaproxy.PandaproxyAPI[0].Address) - require.Exactly(st, 8123, c.Pandaproxy.PandaproxyAPI[0].Port) - }, + name: "fail if deep unrecognized value is passed", + key: "redpanda.unrecognized.name", + value: "foo", + expectErr: true, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Merge(&tt.config) + cfg, err := new(Params).Load(fs) + require.NoError(t, err) + err = cfg.Set(tt.key, tt.value, tt.format) if tt.expectErr { require.Error(t, err) return } require.NoError(t, err) if tt.check != nil { - conf, err := mgr.Get() - require.NoError(t, err) - m, _ := mgr.(*manager) - tt.check(t, conf, m) + tt.check(t, cfg) } }) } @@ -370,53 +311,6 @@ func TestDefault(t *testing.T) { require.Exactly(t, expected, defaultConfig) } -func TestRead(t *testing.T) { - const baseDir string = "/etc/redpanda" - tests := []struct { - name string - path string - before func(afero.Fs, string) error - want func() *Config - wantErr bool - }{ - { - name: "shall return a config struct filled with values from the file", - before: func(fs afero.Fs, path string) error { - bs, err := yaml.Marshal(getValidConfig()) - if err != nil { - return err - } - if err = fs.MkdirAll(baseDir, 0o755); err != nil { - return err - } - _, err = utils.WriteBytes(fs, bs, path) - return err - }, - path: baseDir + "/redpanda.yaml", - want: getValidConfig, - wantErr: false, - }, - // TODO: Add tests for when the config file has missing objects - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := tt.before(fs, tt.path) - require.NoError(t, err) - got, err := mgr.Read(tt.path) - want := tt.want() - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.True(t, filepath.IsAbs(got.ConfigFile)) - require.Exactly(t, want, got) - }) - } -} - func TestWrite(t *testing.T) { tests := []struct { name string @@ -426,103 +320,52 @@ func TestWrite(t *testing.T) { expected string }{ { - name: "it should write the default values for redpanda.kafka_api if it's null", - conf: getValidConfig, - wantErr: false, + name: "write default values", + conf: getValidConfig, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "it should write the default values if redpanda.kafka_api is missing", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without advertised_kafka_api", + name: "write additional values", conf: func() *Config { c := getValidConfig() c.Redpanda.AdvertisedRPCAPI = &SocketAddress{ @@ -531,114 +374,53 @@ schema_registry: {} } return c }, - wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_rpc_api: - address: 174.32.64.2 - port: 33145 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + advertised_rpc_api: + address: 174.32.64.2 + port: 33145 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file without advertised_rpc_api", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdvertisedKafkaAPI = []NamedSocketAddress{{ - Address: "174.32.64.2", - Port: 9092, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - advertised_kafka_api: - - address: 174.32.64.2 - port: 9092 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write a valid config file without an rpk config object", + name: "write if empty struct is passed", conf: func() *Config { c := getValidConfig() c.Rpk = RpkConfig{} @@ -646,393 +428,47 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false -schema_registry: {} -`, - }, - { - name: "shall fail with an invalid config", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.Directory = "" - return c - }, - wantErr: true, - }, - { - name: "shall write a valid config file with an rpk config object", - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall leave unrecognized fields untouched", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - node_id: 1 - - host: - address: 127.0.0.1 - port: 33146 - node_id: 2 - target_quota_byte_rate: 1000000 -unrecognized_top_field: - child: true -`, - conf: getValidConfig, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -unrecognized_top_field: - child: true -`, - }, - { - name: "should merge the new config onto the current one, not just overwrite it", - existingConf: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: func() *Config { - conf := getValidConfig() - conf.Redpanda.SeedServers = []SeedServer{} - conf.Redpanda.Directory = "/different/path" - return conf - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /different/path - default_window_sec: 100 - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.KafkaAPI[0].Name = "outside" - c.Redpanda.KafkaAPITLS = []ServerTLS{{ - Name: "outside", - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - Other: map[string]interface{}{"principal_mapping_rules": "DEFAULT"}, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: false + tune_network: false + tune_disk_scheduler: false + tune_disk_nomerges: false + tune_disk_write_cache: false + tune_disk_irq: false + tune_fstrim: false + tune_cpu: false + tune_aio_events: false + tune_clocksource: false + tune_swappiness: false + tune_transparent_hugepages: false + enable_memory_locking: false + tune_coredump: false + tune_ballast_file: false + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - name: outside - port: 9092 - kafka_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - name: outside - principal_mapping_rules: DEFAULT - require_client_auth: true - truststore_file: /etc/certs/ca.crt - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage schema_registry: {} `, }, { - name: "shall write config with admin tls configuration", - conf: func() *Config { - c := getValidConfig() - c.Redpanda.AdminAPITLS = []ServerTLS{{ - KeyFile: "/etc/certs/admin/cert.key", - TruststoreFile: "/etc/certs/admin/ca.crt", - CertFile: "/etc/certs/admin/cert.crt", - Enabled: true, - RequireClientAuth: true, - }} - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_tls: - - cert_file: /etc/certs/admin/cert.crt - enabled: true - key_file: /etc/certs/admin/cert.key - require_client_auth: true - truststore_file: /etc/certs/admin/ca.crt - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with log_segment_size configuration", + name: "write unrecognized values ('Other' map).", conf: func() *Config { c := getValidConfig() size := 536870912 @@ -1044,469 +480,45 @@ schema_registry: {} }, wantErr: false, expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - log_segment_size: 536870912 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with full pandaproxy configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - PandaproxyAPITLS: []ServerTLS{{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }}, - AdvertisedPandaproxyAPI: []NamedSocketAddress{ - { - Name: "advertised", - Address: "2.3.4.1", - Port: 2341, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - advertised_pandaproxy_api: - - address: 2.3.4.1 - name: advertised - port: 2341 - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 - pandaproxy_api_tls: - - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy api only configuration", - conf: func() *Config { - c := getValidConfig() - c.Pandaproxy = &Pandaproxy{ - PandaproxyAPI: []NamedSocketAddress{ - { - Name: "first", - Address: "1.2.3.4", - Port: 1234, - }, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: - pandaproxy_api: - - address: 1.2.3.4 - name: first - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandaproxy client configuration", - conf: func() *Config { - c := getValidConfig() - c.PandaproxyClient = &KafkaClient{ - Brokers: []SocketAddress{ - { - Address: "1.2.3.4", - Port: 1234, - }, - }, - BrokerTLS: ServerTLS{ - KeyFile: "/etc/certs/cert.key", - TruststoreFile: "/etc/certs/ca.crt", - CertFile: "/etc/certs/cert.crt", - Enabled: true, - RequireClientAuth: true, - }, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -pandaproxy_client: - broker_tls: - cert_file: /etc/certs/cert.crt - enabled: true - key_file: /etc/certs/cert.key - require_client_auth: true - truststore_file: /etc/certs/ca.crt - brokers: - - address: 1.2.3.4 - port: 1234 -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write config with pandproxy client SASL mechanism and SCRAM credentials", - conf: func() *Config { - c := getValidConfig() - mechanism := "abc" - username := "user" - password := "pass" - c.PandaproxyClient = &KafkaClient{ - SASLMechanism: &mechanism, - SCRAMUsername: &username, - SCRAMPassword: &password, - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -pandaproxy_client: - sasl_mechanism: abc - scram_password: pass - scram_username: user -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 -rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "shall write a valid config file with scram configured", - conf: func() *Config { - c := getValidConfig() - c.Rpk.KafkaAPI.SASL = &SASL{ - User: "scram_user", - Password: "scram_password", - Mechanism: "SCRAM-SHA-256", - } - return c - }, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: false - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: - - host: - address: 127.0.0.1 - port: 33145 - - host: - address: 127.0.0.1 - port: 33146 + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: + - address: 127.0.0.1 + port: 33145 + - address: 127.0.0.1 + port: 33146 + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: false + log_segment_size: 536870912 rpk: - coredump_dir: /var/lib/redpanda/coredumps - enable_memory_locking: true - enable_usage_stats: true - kafka_api: - sasl: - password: scram_password - type: SCRAM-SHA-256 - user: scram_user - overprovisioned: false - tune_aio_events: true - tune_ballast_file: true - tune_clocksource: true - tune_coredump: true - tune_cpu: true - tune_disk_irq: true - tune_disk_nomerges: true - tune_disk_scheduler: true - tune_disk_write_cache: true - tune_fstrim: true - tune_network: true - tune_swappiness: true - tune_transparent_hugepages: true - well_known_io: vendor:vm:storage -schema_registry: {} -`, - }, - { - name: "should update an existing config with single kafka_api & advertised_kafka_api obj to a list", - existingConf: `config_file: /etc/redpanda/redpanda.yaml + enable_usage_stats: true + tune_network: true + tune_disk_scheduler: true + tune_disk_nomerges: true + tune_disk_write_cache: true + tune_disk_irq: true + tune_fstrim: true + tune_cpu: true + tune_aio_events: true + tune_clocksource: true + tune_swappiness: true + tune_transparent_hugepages: true + enable_memory_locking: true + tune_coredump: true + coredump_dir: /var/lib/redpanda/coredumps + tune_ballast_file: true + well_known_io: vendor:vm:storage + overprovisioned: false pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - kafka_api: - address: 0.0.0.0 - port: 9092 - advertised_kafka_api: - address: 1.cluster.redpanda.io - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - target_quota_byte_rate: 1000000 -`, - conf: Default, - wantErr: false, - expected: `config_file: /etc/redpanda/redpanda.yaml -pandaproxy: {} -redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - admin_api_doc_dir: /usr/share/redpanda/admin-api-doc - advertised_kafka_api: - - address: 1.cluster.redpanda.io - port: 9092 - auto_create_topics_enabled: true - data_directory: /var/lib/redpanda/data - default_window_sec: 100 - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] - target_quota_byte_rate: 1000000 -rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false schema_registry: {} `, }, @@ -1515,19 +527,12 @@ schema_registry: {} t.Run(tt.name, func(t *testing.T) { path := tt.conf().ConfigFile fs := afero.NewMemMapFs() - mgr := NewManager(fs) if tt.existingConf != "" { _, err := utils.WriteBytes(fs, []byte(tt.existingConf), path) require.NoError(t, err) - } else { - // Write the config file so that WriteConfig backs it up - // when the new one is written. - err := vyaml.Persist(fs, tt.conf(), path) - require.NoError(t, err) } - _, err := mgr.Read(path) - require.NoError(t, err) - err = mgr.Write(tt.conf()) + + err := tt.conf().Write(fs) if tt.wantErr { require.Error(t, err) return @@ -1538,96 +543,6 @@ schema_registry: {} require.NoError(t, err) content := string(contentBytes) require.Equal(t, tt.expected, content) - backup, err := findBackup(fs, filepath.Dir(path)) - require.NoError(t, err) - _, err = fs.Stat(backup) - require.NoError(t, err) - }) - } -} - -func TestWriteLoaded(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - err := mgr.Set( - "rpk.admin", - `[{"address": "192.168.54.2","port": 9092}]`, - "json", - ) - require.NoError(t, err) - - mgr.WriteLoaded() - conf, err := mgr.Get() - require.NoError(t, err) - - newMgr := NewManager(fs) - newConf, err := newMgr.Read(Default().ConfigFile) - require.NoError(t, err) - - require.Exactly(t, conf, newConf) -} - -func TestReadOrGenerate(t *testing.T) { - tests := []struct { - name string - setup func(afero.Fs) error - configFile string - check func(st *testing.T, conf *Config) - expectError bool - }{ - { - name: "it should generate a config file at the given location", - configFile: Default().ConfigFile, - }, - { - name: "it shouldn't fail if there's a config file already", - setup: func(fs afero.Fs) error { - conf := Default() - mgr := NewManager(fs) - return mgr.Write(conf) - }, - configFile: Default().ConfigFile, - }, - { - name: "it should fail if the existing config file's content isn't valid yaml", - setup: func(fs afero.Fs) error { - bs := []byte(`redpanda: -- something`) - return vyaml.Persist(fs, bs, Default().ConfigFile) - }, - configFile: Default().ConfigFile, - expectError: true, - }, - { - name: "it should set config_file to the right value", - configFile: "./redpanda.yaml", - check: func(st *testing.T, conf *Config) { - path, err := filepath.Abs("./redpanda.yaml") - require.NoError(st, err) - require.Exactly(st, conf.ConfigFile, path) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - if tt.setup != nil { - err := tt.setup(fs) - require.NoError(t, err) - } - _, err := readOrGenerate(fs, InitViper(fs), tt.configFile) - if tt.expectError { - require.Error(t, err) - return - } - require.NoError(t, err) - conf, err := mgr.Read(tt.configFile) - require.NoError(t, err) - if tt.check != nil { - tt.check(t, conf) - } }) } } @@ -1816,7 +731,7 @@ func TestCheckConfig(t *testing.T) { name: "shall return an error when one of the seed servers' address is empty", conf: func() *Config { c := getValidConfig() - c.Redpanda.SeedServers[0].Host.Address = "" + c.Redpanda.SeedServers[0].Address = "" return c }, expected: []string{"redpanda.seed_servers.0.host.address can't be empty"}, @@ -1825,7 +740,7 @@ func TestCheckConfig(t *testing.T) { name: "shall return an error when one of the seed servers' port is 0", conf: func() *Config { c := getValidConfig() - c.Redpanda.SeedServers[1].Host.Port = 0 + c.Redpanda.SeedServers[1].Port = 0 return c }, expected: []string{"redpanda.seed_servers.1.host.port can't be 0"}, @@ -1849,8 +764,7 @@ func TestCheckConfig(t *testing.T) { c.Rpk.CoredumpDir = "" return c }, - expected: []string{"if rpk.tune_coredump is set to true," + - "rpk.coredump_dir can't be empty"}, + expected: []string{"if rpk.tune_coredump is set to true, rpk.coredump_dir can't be empty"}, }, { name: "shall return no error if setup is empty," + @@ -1865,7 +779,7 @@ func TestCheckConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, got := Check(tt.conf()) + _, got := tt.conf().Check() errMsgs := []string{} for _, err := range got { errMsgs = append(errMsgs, err.Error()) @@ -1874,31 +788,3 @@ func TestCheckConfig(t *testing.T) { }) } } - -func TestWriteAndGenerateNodeUuid(t *testing.T) { - fs := afero.NewMemMapFs() - mgr := NewManager(fs) - baseDir := "/etc/redpanda" - path := baseDir + "/redpanda.yaml" - conf := getValidConfig() - conf.ConfigFile = path - bs, err := yaml.Marshal(conf) - require.NoError(t, err) - err = fs.MkdirAll(baseDir, 0o755) - require.NoError(t, err) - _, err = utils.WriteBytes(fs, bs, path) - require.NoError(t, err) - err = mgr.WriteNodeUUID(conf) - require.NoError(t, err) - require.NotEqual(t, "", conf.NodeUUID) - readConf, err := mgr.Read(path) - require.NoError(t, err) - require.Exactly(t, conf, readConf) -} - -func TestGet(t *testing.T) { - mgr := NewManager(afero.NewMemMapFs()) - conf, err := mgr.Get() - require.NoError(t, err) - require.Exactly(t, Default(), conf) -} diff --git a/src/go/rpk/pkg/config/find.go b/src/go/rpk/pkg/config/find.go deleted file mode 100644 index 4b2147b72b3f6..0000000000000 --- a/src/go/rpk/pkg/config/find.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" -) - -func FindConfigFile(fs afero.Fs) (string, error) { - log.Debugf("Looking for the redpanda config file") - configPathProviders := []func() ([]string, error){ - currentDirectory, - sysConfDirectory, - homeDirectory, - } - - lookedUpPaths := []string{} - for _, provider := range configPathProviders { - paths, err := provider() - if err != nil { - return "", err - } - for _, path := range paths { - candidate := filepath.Join(path, "redpanda.yaml") - lookedUpPaths = append(lookedUpPaths, candidate) - log.Debugf("Looking for redpanda config file in '%s'", path) - if exists, _ := afero.Exists(fs, candidate); exists { - return candidate, nil - } - } - } - // os.PathError can be checked with os.IsNotExist. - return "", &os.PathError{ - Op: "Open", - Path: strings.Join(lookedUpPaths, ", "), - Err: os.ErrNotExist, - } -} - -func sysConfDirectory() ([]string, error) { - return []string{"/etc/redpanda"}, nil -} - -func currentDirectory() ([]string, error) { - currentDir, err := os.Getwd() - if err != nil { - return nil, err - } - return []string{currentDir}, nil -} - -func homeDirectory() ([]string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, err - } - return []string{homeDir}, nil -} diff --git a/src/go/rpk/pkg/config/find_test.go b/src/go/rpk/pkg/config/find_test.go deleted file mode 100644 index bf6dbeb730582..0000000000000 --- a/src/go/rpk/pkg/config/find_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "os" - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -func TestFindConfig(t *testing.T) { - tests := []struct { - name string - before func(fs afero.Fs) - want string - wantErr bool - }{ - { - name: "should return an error when config is not found", - before: func(afero.Fs) {}, - want: "", - wantErr: true, - }, - { - name: "should return config file from home directory", - before: func(fs afero.Fs) { - createConfigIn(fs, homeDir()) - }, - want: filepath.Join(homeDir(), "redpanda.yaml"), - }, - { - name: "should return config file from 'etc' directory", - before: func(fs afero.Fs) { - createConfigIn(fs, filepath.Join("/", "etc", "redpanda")) - }, - want: filepath.Join("/", "etc", "redpanda", "redpanda.yaml"), - }, - { - name: "should return config file from current directory", - before: func(fs afero.Fs) { - createConfigIn(fs, currentDir()) - }, - want: filepath.Join(currentDir(), "redpanda.yaml"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - tt.before(fs) - got, err := FindConfigFile(fs) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, tt.want, got) - }) - } -} - -func createConfigIn(fs afero.Fs, path string) { - fs.Create(filepath.Join(path, "redpanda.yaml")) -} - -func currentDir() string { - d, _ := os.Getwd() - return d -} - -func homeDir() string { - d, _ := os.UserHomeDir() - return d -} diff --git a/src/go/rpk/pkg/config/license.go b/src/go/rpk/pkg/config/license.go deleted file mode 100644 index 5ecf38d923de0..0000000000000 --- a/src/go/rpk/pkg/config/license.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "hash/crc32" - "time" - - log "github.com/sirupsen/logrus" -) - -type LicenseKey struct { - Organization string `json:"o"` - ExpirationYear uint16 `json:"y"` - ExpirationMonth uint8 `json:"m"` - ExpirationDay uint8 `json:"d"` - Checksum uint32 `json:"c"` -} - -// Checks the key and prints a warning if the key is invalid. -func CheckAndPrintNotice(key string) { - err := CheckLicenseKey(key) - if err != nil { - log.Info(err) - return - } -} - -// Checks the license key, which is a Base64-encoded JSON object with the the -// organization it was granted to, the expiration date, and the hash of the -// concatenation of those fields. -// Returns true if the exp. date hasn't passed yet. Returns false otherwise. -// Returns an error if the key isn't base64-encoded or if it's not valid JSON. -func CheckLicenseKey(key string) error { - msg := "Please get a new one at https://vectorized.io/download-trial/" - if key == "" { - return errors.New("Missing license key. " + msg) - } - decoded, err := base64.StdEncoding.DecodeString(key) - invalidErr := errors.New("Invalid license key. " + msg) - if err != nil { - return invalidErr - } - lk := &LicenseKey{} - err = json.Unmarshal(decoded, lk) - if err != nil { - return invalidErr - } - - content := fmt.Sprintf( - "%s%d%d%d", - lk.Organization, - lk.ExpirationYear, - lk.ExpirationMonth, - lk.ExpirationDay, - ) - - checksum := crc32.ChecksumIEEE([]byte(content)) - - if lk.Checksum != checksum { - return invalidErr - } - - expDate := time.Date( - int(lk.ExpirationYear), - time.Month(lk.ExpirationMonth), - int(lk.ExpirationDay), - 0, 0, 0, 0, - time.UTC, - ) - - if time.Now().After(expDate) { - return errors.New("Your license key has expired. " + msg) - } - return nil -} diff --git a/src/go/rpk/pkg/config/license_test.go b/src/go/rpk/pkg/config/license_test.go deleted file mode 100644 index 20b5fd9df80de..0000000000000 --- a/src/go/rpk/pkg/config/license_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config_test - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "hash/crc32" - "testing" - "time" - - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/config" - "github.com/stretchr/testify/require" -) - -const vectorizedOrg string = "vectorized.io" - -func makeKey(org string, exp time.Time, overrideHash uint32) (string, error) { - content := fmt.Sprintf( - "%s%d%d%d", - org, - exp.Year(), - exp.Month(), - exp.Day(), - ) - var checksum uint32 - if overrideHash != 0 { - checksum = overrideHash - } else { - checksum = crc32.ChecksumIEEE([]byte(content)) - } - lk := &config.LicenseKey{ - Organization: org, - ExpirationYear: uint16(exp.Year()), - ExpirationMonth: uint8(exp.Month()), - ExpirationDay: uint8(exp.Day()), - Checksum: checksum, - } - bs, err := json.Marshal(lk) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(bs), nil -} - -func TestCheckLicenseKey(t *testing.T) { - msg := "Please get a new one at https://vectorized.io/download-trial/" - tests := []struct { - name string - key func() (string, error) - expected string - }{ - { - name: "key is valid if the hashes match and the expiration date hasn't passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "", - }, - { - name: "key is invalid if the hashes don't match", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(48 * time.Hour) - return makeKey(organization, expiresAt, 123) - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if the expiration date has already passed", - key: func() (string, error) { - organization := vectorizedOrg - expiresAt := time.Now().Add(-48 * time.Hour) - return makeKey(organization, expiresAt, 0) - }, - expected: "Your license key has expired. " + msg, - }, - { - name: "key is invalid if it's not base64-encoded", - key: func() (string, error) { - return "this is defs not base 64", nil - }, - expected: "Invalid license key. " + msg, - }, - { - name: "key is invalid if it's not valid base64-encoded JSON", - key: func() (string, error) { - jsonStr := []byte("{\"thisJson\":\"is invalid\"") - return base64.StdEncoding.EncodeToString(jsonStr), nil - }, - expected: "Invalid license key. " + msg, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(st *testing.T) { - key, err := tt.key() - require.NoError(st, err) - err = config.CheckLicenseKey(key) - if tt.expected == "" { - require.Nil(st, err) - return - } - require.EqualError(st, err, tt.expected) - }) - } -} diff --git a/src/go/rpk/pkg/config/manager.go b/src/go/rpk/pkg/config/manager.go deleted file mode 100644 index a8a098b1fc718..0000000000000 --- a/src/go/rpk/pkg/config/manager.go +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package config - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - - "github.com/google/uuid" - "github.com/mitchellh/mapstructure" - "github.com/redpanda-data/redpanda/src/go/rpk/pkg/utils" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" -) - -type Manager interface { - // Reads the config from the given path - Read(path string) (*Config, error) - // Writes the config to Config.ConfigFile - Write(conf *Config) error - // Writes the currently-loaded config to redpanda.config_file - WriteLoaded() error - // Get the currently-loaded config - Get() (*Config, error) - // Sets key to the given value (parsing it according to the format) - Set(key, value, format string) error - // If path is empty, tries to find the file in the default locations. - // Otherwise, it tries to read the file and load it. If the file doesn't - // exist, it tries to create it with the default configuration. - FindOrGenerate(path string) (*Config, error) - // Tries reading a config file at the given path, or tries to find it in - // the default locations if it doesn't exist. - ReadOrFind(path string) (*Config, error) - // Generates and writes the node's UUID - WriteNodeUUID(conf *Config) error - // Merges an input config to the currently-loaded map - Merge(conf *Config) error -} - -type manager struct { - fs afero.Fs - v *viper.Viper -} - -func NewManager(fs afero.Fs) Manager { - return &manager{fs, InitViper(fs)} -} - -func (m *manager) FindOrGenerate(path string) (*Config, error) { - if path == "" { - addConfigPaths(m.v) - err := m.v.ReadInConfig() - if err == nil { - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - if !notFound { - return nil, err - } - path = Default().ConfigFile - } - return readOrGenerate(m.fs, m.v, path) -} - -// Tries reading a config file at the given path, or generates a default config -// and writes it to the path. -func readOrGenerate(fs afero.Fs, v *viper.Viper, path string) (*Config, error) { - abs, err := absPath(path) - if err != nil { - return nil, err - } - v.SetConfigFile(abs) - err = v.ReadInConfig() - if err == nil { - // The config file's there, there's nothing to do. - return unmarshal(v) - } - _, notFound := err.(viper.ConfigFileNotFoundError) //nolint:errorlint // Viper returns a non-pointer error https://github.com/spf13/viper/issues/1139 - notExist := os.IsNotExist(err) - if err != nil && !notFound && !notExist { - return nil, fmt.Errorf( - "An error happened while trying to read %s: %v", - abs, - err, - ) - } - log.Debug(err) - log.Infof( - "Couldn't find config file at %s. Generating it.", - abs, - ) - v.Set("config_file", abs) - err = createConfigDir(fs, abs) - if err != nil { - return nil, err - } - err = v.WriteConfigAs(abs) - if err != nil { - return nil, fmt.Errorf( - "Couldn't write config to %s: %v", - abs, - err, - ) - } - return unmarshal(v) -} - -func (m *manager) ReadOrFind(path string) (*Config, error) { - var err error - if path == "" { - path, err = FindConfigFile(m.fs) - if err != nil { - return nil, err - } - } - return m.Read(path) -} - -func (m *manager) Read(path string) (*Config, error) { - // If the path was set, try reading only from there. - abs, err := filepath.Abs(path) - if err != nil { - return nil, err - } - m.v.SetConfigFile(abs) - err = m.v.ReadInConfig() - if err != nil { - return nil, err - } - conf, err := unmarshal(m.v) - if err != nil { - return nil, err - } - conf.ConfigFile, err = absPath(m.v.ConfigFileUsed()) - return conf, err -} - -func (m *manager) WriteNodeUUID(conf *Config) error { - id, err := uuid.NewUUID() - if err != nil { - return err - } - conf.NodeUUID = id.String() - return m.Write(conf) -} - -func (m *manager) Get() (*Config, error) { - return unmarshal(m.v) -} - -// Checks config and writes it to the given path. -func (m *manager) Write(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - // Merge the config into a new viper.Viper instance to prevent - // concurrent writes to the underlying config map. - v := InitViper(m.fs) - current, err := unmarshal(m.v) - if err != nil { - return err - } - currentMap, err := toMap(current) - if err != nil { - return err - } - v.MergeConfigMap(currentMap) - v.MergeConfigMap(confMap) - return checkAndWrite(m.fs, v, conf.ConfigFile) -} - -// Writes the currently loaded config. -func (m *manager) WriteLoaded() error { - return checkAndWrite(m.fs, m.v, m.v.GetString("config_file")) -} - -func write(fs afero.Fs, v *viper.Viper, path string) error { - err := createConfigDir(fs, path) - if err != nil { - return err - } - err = v.WriteConfigAs(path) - if err != nil { - return err - } - log.Debugf( - "Configuration written to %s.", - path, - ) - return nil -} - -func (m *manager) setDeduceFormat(key, value string) error { - replace := func(key string, newValue interface{}) error { - newV := viper.New() - newV.Set(key, newValue) - return m.v.MergeConfigMap(newV.AllSettings()) - } - - var newVal interface{} - switch { - case json.Unmarshal([]byte(value), &newVal) == nil: // Try JSON - return replace(key, newVal) - - case yaml.Unmarshal([]byte(value), &newVal) == nil: // Try YAML - return replace(key, newVal) - - default: // Treat the value as a "single" - m.v.Set(key, parse(value)) - return nil - } -} - -func (m *manager) Set(key, value, format string) error { - if key == "" { - return errors.New("empty config field key") - } - if format == "" { - return m.setDeduceFormat(key, value) - } - var newConfValue interface{} - switch strings.ToLower(format) { - case "single": - m.v.Set(key, parse(value)) - return nil - case "yaml": - err := yaml.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - case "json": - err := json.Unmarshal([]byte(value), &newConfValue) - if err != nil { - return err - } - default: - return fmt.Errorf("unsupported format %s", format) - } - newV := viper.New() - newV.Set(key, newConfValue) - return m.v.MergeConfigMap(newV.AllSettings()) -} - -func (m *manager) Merge(conf *Config) error { - confMap, err := toMap(conf) - if err != nil { - return err - } - return m.v.MergeConfigMap(confMap) -} - -func checkAndWrite(fs afero.Fs, v *viper.Viper, path string) error { - ok, errs := check(v) - if !ok { - reasons := []string{} - for _, err := range errs { - reasons = append(reasons, err.Error()) - } - return errors.New(strings.Join(reasons, ", ")) - } - lastBackupFile, err := findBackup(fs, filepath.Dir(path)) - if err != nil { - return err - } - exists, err := afero.Exists(fs, path) - if err != nil { - return err - } - if !exists { - // If the config doesn't exist, just write it. - return write(fs, v, path) - } - // Otherwise, backup the current config file, write the new one, and - // try to recover if there's an error. - log.Debug("Backing up the current config") - backup, err := utils.BackupFile(fs, path) - if err != nil { - return err - } - log.Debugf("Backed up the current config to %s", backup) - if lastBackupFile != "" && lastBackupFile != backup { - log.Debug("Removing previous backup file") - err = fs.Remove(lastBackupFile) - if err != nil { - return err - } - } - log.Debugf("Writing the new redpanda config to '%s'", path) - err = write(fs, v, path) - if err != nil { - return recover(fs, backup, path, err) //nolint:revive // false positive: this recover function is different from built-in recover - } - return nil -} - -func recover(fs afero.Fs, backup, path string, err error) error { - log.Infof("Recovering the previous confing from %s", backup) - recErr := utils.CopyFile(fs, backup, path) - if recErr != nil { - msg := "couldn't persist the new config due to '%v'," + - " nor recover the backup due to '%v" - return fmt.Errorf(msg, err, recErr) - } - return fmt.Errorf("couldn't persist the new config due to '%v'", err) -} - -func unmarshal(v *viper.Viper) (*Config, error) { - result := &Config{} - decoderConfig := decoderConfig() - decoderConfig.Result = result - decoder, err := mapstructure.NewDecoder(&decoderConfig) - if err != nil { - return nil, err - } - err = decoder.Decode(v.AllSettings()) - if err != nil { - return nil, err - } - if result.ConfigFile == "" { - result.ConfigFile, err = absPath(v.ConfigFileUsed()) - if err != nil { - return nil, err - } - } - return result, nil -} - -// Redpanda version < 21.1.4 only supported a single anonymous listener and a -// single anonymous advertised address. This custom decode function translates -// a single SocketAddress-equivalent map[string]interface{} into a -// []NamedSocketAddress. -func v21_1_4MapToNamedSocketAddressSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]NamedSocketAddress{}) { - if from.Kind() == reflect.Map { - sa := NamedSocketAddress{} - err := mapstructure.Decode(data, &sa) - if err != nil { - return nil, err - } - return []NamedSocketAddress{sa}, nil - } - } - return data, nil -} - -// Redpanda version <= 21.4.1 only supported a single TLS config. This custom -// decode function translates a single TLS config-equivalent -// map[string]interface{} into a []ServerTLS. -//nolint:revive // using underscore here is intended -func v21_4_1TlsMapToNamedTlsSlice( - from, to reflect.Type, data interface{}, -) (interface{}, error) { - if to == reflect.TypeOf([]ServerTLS{}) { - if from.Kind() == reflect.Map { - tls := ServerTLS{} - err := mapstructure.Decode(data, &tls) - if err != nil { - return nil, err - } - return []ServerTLS{tls}, nil - } - } - return data, nil -} - -func parse(val string) interface{} { - if i, err := strconv.Atoi(val); err == nil { - return i - } - if f, err := strconv.ParseFloat(val, 64); err == nil { - return f - } - if b, err := strconv.ParseBool(val); err == nil { - return b - } - return val -} - -func absPath(path string) (string, error) { - absPath, err := filepath.Abs(path) - if err != nil { - return "", fmt.Errorf( - "couldn't convert the used config file path to"+ - " absolute: %s", - path, - ) - } - return absPath, nil -} - -func createConfigDir(fs afero.Fs, configFile string) error { - dir := filepath.Dir(configFile) - err := fs.MkdirAll(dir, 0o755) - if err != nil { - return fmt.Errorf( - "couldn't create config dir %s: %v", - dir, - err, - ) - } - return nil -} diff --git a/src/go/rpk/pkg/config/params.go b/src/go/rpk/pkg/config/params.go index ed701b43e7763..8d247d9e3dba1 100644 --- a/src/go/rpk/pkg/config/params.go +++ b/src/go/rpk/pkg/config/params.go @@ -10,11 +10,13 @@ package config import ( + "encoding/json" "errors" "fmt" "net" "os" "path/filepath" + "reflect" "strconv" "strings" "syscall" @@ -23,7 +25,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // This file contains the Params type, which will eventually be created in @@ -210,7 +212,7 @@ func ParamsFromCommand(cmd *cobra.Command) *Params { } val := f.Value.String() - // Value.String() adds backets to slice types, and we + // Value.String() adds brackets to slice types, and we // need to strip that here. if stripBrackets { if len(val) > 0 && val[0] == '[' && val[len(val)-1] == ']' { @@ -234,8 +236,20 @@ func ParamsFromCommand(cmd *cobra.Command) *Params { // * Sets unset default values. // func (p *Params) Load(fs afero.Fs) (*Config, error) { + cf := "/etc/redpanda/redpanda.yaml" + // If we have a config path loaded (through --config flag) the user + // expect to load or create the file from this directory. + if p.ConfigPath != "" { + if exist, _ := afero.Exists(fs, p.ConfigPath); !exist { + err := fs.MkdirAll(filepath.Dir(p.ConfigPath), 0o755) + if err != nil { + return nil, err + } + } + cf = p.ConfigPath + } c := &Config{ - ConfigFile: "/etc/redpanda/redpanda.yaml", + ConfigFile: cf, Redpanda: RedpandaConfig{ Directory: "/var/lib/redpanda/data", RPCServer: SocketAddress{ @@ -255,6 +269,9 @@ func (p *Params) Load(fs afero.Fs) (*Config, error) { Rpk: RpkConfig{ CoredumpDir: "/var/lib/redpanda/coredump", }, + // enable pandaproxy and schema_registry by default + Pandaproxy: &Pandaproxy{}, + SchemaRegistry: &SchemaRegistry{}, } if err := p.readConfig(fs, c); err != nil { @@ -276,9 +293,8 @@ func (p *Params) Load(fs afero.Fs) (*Config, error) { func (c *Config) Write(fs afero.Fs) (rerr error) { cfgPath := c.loadedPath if cfgPath == "" { - cfgPath = Default().ConfigFile + cfgPath = c.ConfigFile } - b, err := yaml.Marshal(c) if err != nil { return fmt.Errorf("marshal error in loaded config, err: %s", err) @@ -552,3 +568,133 @@ func (c *Config) addUnsetDefaults() { r.AdminAPI.Addresses = []string{"127.0.0.1:9644"} } } + +// Set allow to set a single configuration property by passing a key value pair +// +// Key: string containing the yaml property tag, e.g: 'rpk.admin_api'. +// Value: string representation of the value, either single value or partial +// representation if a format is passed (yaml / json). +// Format: either single, json, or yaml (default: single). +func (c *Config) Set(key, value, format string) error { + if key == "" { + return fmt.Errorf("key field must not be empty") + } + props := strings.Split(key, ".") + rv := reflect.ValueOf(c).Elem() + found, other, err := getField(props, rv) + if err != nil { + return err + } + isOther := other != reflect.Value{} + + field := found + if isOther { + field = other + } + + if field.CanAddr() { + i := field.Addr().Interface() + in := value + switch strings.ToLower(format) { + case "yaml", "single", "": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("%s: %s", p, value) + } + err = yaml.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + case "json": + if isOther { + p := props[len(props)-1] + in = fmt.Sprintf("{%q: %q}", p, value) + } + err = json.Unmarshal([]byte(in), i) + if err != nil { + return err + } + return nil + default: + return fmt.Errorf("unsupported format %s", format) + } + } + return errors.New("rpk bug, please describe how you encountered this at https://github.com/redpanda-data/redpanda/issues/new?assignees=&labels=kind%2Fbug&template=01_bug_report.md") +} + +// getField deeply search in p for the value that reflect property props. +func getField(props []string, p reflect.Value) (reflect.Value, reflect.Value, error) { + if len(props) == 0 { + return p, reflect.Value{}, nil + } + if p.Kind() == reflect.Slice && p.Len() > 0 { + p = p.Index(0) + } + if p.Kind() == reflect.Slice && p.Len() == 0 { + p.Set(reflect.New(p.Type().Elem())) + } + // If is a nil pointer we assign the zero value, and we reassign p to the + // value that p points to + if p.Kind() == reflect.Ptr { + if p.IsNil() { + p.Set(reflect.New(p.Type().Elem())) + } + p = reflect.Indirect(p) + } + if p.Kind() == reflect.Struct { + newP, other, err := getFieldByTag(props[0], p) + if err != nil { + return reflect.Value{}, reflect.Value{}, err + } + // if is "Other" map field, we stop the recursion and return + if (other != reflect.Value{}) { + // user may try to set deep unmanaged property: + // rpk.unmanaged.name = "name" + if len(props) > 1 { + return reflect.Value{}, reflect.Value{}, fmt.Errorf("cannot set property %v", strings.Join(props, ".")) + } + return reflect.Value{}, other, nil + } + return getField(props[1:], newP) + } + return reflect.Value{}, reflect.Value{}, fmt.Errorf("cannot recurse on type %v", p.Type()) +} + +// getFieldByTag finds a field with a given yaml tag and returns 3 parameters: +// +// 1. if tag is found within the struct, return the field. +// 2. if tag is not found _but_ the struct has "Other" field, return Other. +// 3. Error if it can't find the given tag and "Other" field is unavailable. +func getFieldByTag(tag string, p reflect.Value) (reflect.Value, reflect.Value, error) { + t := p.Type() + var other bool + // Loop struct to get the field that match tag. + for i := 0; i < p.NumField(); i++ { + // Rpk blindly set configuration parameters, we parse these parameters + // in the "Other" field. + if t.Field(i).Name == "Other" { + other = true + } + yt := t.Field(i).Tag.Get("yaml") + + // yaml struct tags can contain flags such as omitempty, + // when tag.Get("yaml") is called it will return + // "my_tag,omitempty" + // so we only need first parameter of the string slice. + ft := strings.Split(yt, ",")[0] + + if ft == tag { + return p.Field(i), reflect.Value{}, nil + } else { + continue + } + } + + // If we can't find the tag but the struct has an 'Other' map field: + if other { + return reflect.Value{}, p.FieldByName("Other"), nil + } + + return reflect.Value{}, reflect.Value{}, fmt.Errorf("couldn't find property %q", tag) +} diff --git a/src/go/rpk/pkg/config/params_test.go b/src/go/rpk/pkg/config/params_test.go index e8528bc7883f3..dc345462ca564 100644 --- a/src/go/rpk/pkg/config/params_test.go +++ b/src/go/rpk/pkg/config/params_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/spf13/afero" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func TestParams_Write(t *testing.T) { @@ -20,27 +20,27 @@ func TestParams_Write(t *testing.T) { name: "create default config file if there is no config file yet", exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: /var/lib/redpanda/data - node_id: 0 - seed_servers: [] - rpc_server: - address: 0.0.0.0 - port: 33145 - kafka_api: - - address: 0.0.0.0 - port: 9092 - admin: - - address: 0.0.0.0 - port: 9644 - developer_mode: true + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - kafka_api: - brokers: - - 0.0.0.0:9092 - admin_api: - addresses: - - 127.0.0.1:9644 - enable_usage_stats: false + kafka_api: + brokers: + - 0.0.0.0:9092 + admin_api: + addresses: + - 127.0.0.1:9644 + enable_usage_stats: false `, }, { @@ -49,8 +49,8 @@ rpk: cfgToWrite: &Config{ConfigFile: "/etc/redpanda/redpanda.yaml", Redpanda: RedpandaConfig{ID: 6}}, exp: `config_file: /etc/redpanda/redpanda.yaml redpanda: - data_directory: "" - node_id: 6 + data_directory: "" + node_id: 6 `, }, } diff --git a/src/go/rpk/pkg/config/schema.go b/src/go/rpk/pkg/config/schema.go index fb5f9aa629152..c3834b3e948e9 100644 --- a/src/go/rpk/pkg/config/schema.go +++ b/src/go/rpk/pkg/config/schema.go @@ -21,19 +21,19 @@ type Config struct { file *Config loadedPath string - NodeUUID string `yaml:"node_uuid,omitempty" mapstructure:"node_uuid,omitempty" json:"nodeUuid"` - Organization string `yaml:"organization,omitempty" mapstructure:"organization,omitempty" json:"organization"` - LicenseKey string `yaml:"license_key,omitempty" mapstructure:"license_key,omitempty" json:"licenseKey"` - ClusterID string `yaml:"cluster_id,omitempty" mapstructure:"cluster_id,omitempty" json:"clusterId"` - ConfigFile string `yaml:"config_file" mapstructure:"config_file" json:"configFile"` - Redpanda RedpandaConfig `yaml:"redpanda" mapstructure:"redpanda" json:"redpanda"` - Rpk RpkConfig `yaml:"rpk" mapstructure:"rpk" json:"rpk"` - Pandaproxy *Pandaproxy `yaml:"pandaproxy,omitempty" mapstructure:"pandaproxy,omitempty" json:"pandaproxy,omitempty"` - PandaproxyClient *KafkaClient `yaml:"pandaproxy_client,omitempty" mapstructure:"pandaproxy_client,omitempty" json:"pandaproxyClient,omitempty"` - SchemaRegistry *SchemaRegistry `yaml:"schema_registry,omitempty" mapstructure:"schema_registry,omitempty" json:"schemaRegistry,omitempty"` - SchemaRegistryClient *KafkaClient `yaml:"schema_registry_client,omitempty" mapstructure:"schema_registry_client,omitempty" json:"schemaRegistryClient,omitempty"` + NodeUUID string `yaml:"node_uuid,omitempty" json:"nodeUuid"` + Organization string `yaml:"organization,omitempty" json:"organization"` + LicenseKey string `yaml:"license_key,omitempty" json:"licenseKey"` + ClusterID string `yaml:"cluster_id,omitempty" json:"clusterId"` + ConfigFile string `yaml:"config_file" json:"configFile"` + Redpanda RedpandaConfig `yaml:"redpanda" json:"redpanda"` + Rpk RpkConfig `yaml:"rpk" json:"rpk"` + Pandaproxy *Pandaproxy `yaml:"pandaproxy,omitempty" json:"pandaproxy,omitempty"` + PandaproxyClient *KafkaClient `yaml:"pandaproxy_client,omitempty" json:"pandaproxyClient,omitempty"` + SchemaRegistry *SchemaRegistry `yaml:"schema_registry,omitempty" json:"schemaRegistry,omitempty"` + SchemaRegistryClient *KafkaClient `yaml:"schema_registry_client,omitempty" json:"schemaRegistryClient,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Other map[string]interface{} `yaml:",inline"` } // File returns the configuration as read from a file, with no defaults @@ -44,67 +44,68 @@ func (c *Config) File() *Config { } type RedpandaConfig struct { - Directory string `yaml:"data_directory" mapstructure:"data_directory" json:"dataDirectory"` - ID int `yaml:"node_id" mapstructure:"node_id" json:"id"` - Rack string `yaml:"rack,omitempty" mapstructure:"rack" json:"rack"` - SeedServers []SeedServer `yaml:"seed_servers" mapstructure:"seed_servers" json:"seedServers"` - RPCServer SocketAddress `yaml:"rpc_server" mapstructure:"rpc_server" json:"rpcServer"` - RPCServerTLS []ServerTLS `yaml:"rpc_server_tls,omitempty" mapstructure:"rpc_server_tls,omitempty" json:"rpcServerTls"` - KafkaAPI []NamedSocketAddress `yaml:"kafka_api" mapstructure:"kafka_api" json:"kafkaApi"` - KafkaAPITLS []ServerTLS `yaml:"kafka_api_tls,omitempty" mapstructure:"kafka_api_tls,omitempty" json:"kafkaApiTls"` - AdminAPI []NamedSocketAddress `yaml:"admin" mapstructure:"admin" json:"admin"` - AdminAPITLS []ServerTLS `yaml:"admin_api_tls,omitempty" mapstructure:"admin_api_tls,omitempty" json:"adminApiTls"` - CoprocSupervisorServer SocketAddress `yaml:"coproc_supervisor_server,omitempty" mapstructure:"coproc_supervisor_server" json:"coprocSupervisorServer"` - AdminAPIDocDir string `yaml:"admin_api_doc_dir,omitempty" mapstructure:"admin_api_doc_dir" json:"adminApiDocDir"` - DashboardDir string `yaml:"dashboard_dir,omitempty" mapstructure:"dashboard_dir" json:"dashboardDir"` - CloudStorageCacheDirectory string `yaml:"cloud_storage_cache_directory,omitempty" mapstructure:"cloud_storage_cache_directory" json:"CloudStorageCacheDirectory"` - AdvertisedRPCAPI *SocketAddress `yaml:"advertised_rpc_api,omitempty" mapstructure:"advertised_rpc_api,omitempty" json:"advertisedRpcApi,omitempty"` - AdvertisedKafkaAPI []NamedSocketAddress `yaml:"advertised_kafka_api,omitempty" mapstructure:"advertised_kafka_api,omitempty" json:"advertisedKafkaApi,omitempty"` - DeveloperMode bool `yaml:"developer_mode" mapstructure:"developer_mode" json:"developerMode"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Directory string `yaml:"data_directory" json:"dataDirectory"` + ID int `yaml:"node_id" json:"id"` + Rack string `yaml:"rack,omitempty" json:"rack"` + SeedServers []SeedServer `yaml:"seed_servers" json:"seedServers"` + RPCServer SocketAddress `yaml:"rpc_server" json:"rpcServer"` + RPCServerTLS []ServerTLS `yaml:"rpc_server_tls,omitempty" json:"rpcServerTls"` + KafkaAPI []NamedSocketAddress `yaml:"kafka_api" json:"kafkaApi"` + KafkaAPITLS []ServerTLS `yaml:"kafka_api_tls,omitempty" json:"kafkaApiTls"` + AdminAPI []NamedSocketAddress `yaml:"admin" json:"admin"` + AdminAPITLS []ServerTLS `yaml:"admin_api_tls,omitempty" json:"adminApiTls"` + CoprocSupervisorServer SocketAddress `yaml:"coproc_supervisor_server,omitempty" json:"coprocSupervisorServer"` + AdminAPIDocDir string `yaml:"admin_api_doc_dir,omitempty" json:"adminApiDocDir"` + DashboardDir string `yaml:"dashboard_dir,omitempty" json:"dashboardDir"` + CloudStorageCacheDirectory string `yaml:"cloud_storage_cache_directory,omitempty" json:"CloudStorageCacheDirectory"` + AdvertisedRPCAPI *SocketAddress `yaml:"advertised_rpc_api,omitempty" json:"advertisedRpcApi,omitempty"` + AdvertisedKafkaAPI []NamedSocketAddress `yaml:"advertised_kafka_api,omitempty" json:"advertisedKafkaApi,omitempty"` + DeveloperMode bool `yaml:"developer_mode" json:"developerMode"` + Other map[string]interface{} `yaml:",inline"` } type Pandaproxy struct { - PandaproxyAPI []NamedSocketAddress `yaml:"pandaproxy_api,omitempty" mapstructure:"pandaproxy_api,omitempty" json:"pandaproxyApi,omitempty"` - PandaproxyAPITLS []ServerTLS `yaml:"pandaproxy_api_tls,omitempty" mapstructure:"pandaproxy_api_tls,omitempty" json:"pandaproxyApiTls,omitempty"` - AdvertisedPandaproxyAPI []NamedSocketAddress `yaml:"advertised_pandaproxy_api,omitempty" mapstructure:"advertised_pandaproxy_api,omitempty" json:"advertisedPandaproxyApi,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + PandaproxyAPI []NamedSocketAddress `yaml:"pandaproxy_api,omitempty" json:"pandaproxyApi,omitempty"` + PandaproxyAPITLS []ServerTLS `yaml:"pandaproxy_api_tls,omitempty" json:"pandaproxyApiTls,omitempty"` + AdvertisedPandaproxyAPI []NamedSocketAddress `yaml:"advertised_pandaproxy_api,omitempty" json:"advertisedPandaproxyApi,omitempty"` + Other map[string]interface{} `yaml:",inline"` } type SchemaRegistry struct { - SchemaRegistryAPI []NamedSocketAddress `yaml:"schema_registry_api,omitempty" mapstructure:"schema_registry_api,omitempty" json:"schemaRegistryApi,omitempty"` - SchemaRegistryAPITLS []ServerTLS `yaml:"schema_registry_api_tls,omitempty" mapstructure:"schema_registry_api_tls,omitempty" json:"schemaRegistryApiTls,omitempty"` - SchemaRegistryReplicationFactor *int `yaml:"schema_registry_replication_factor,omitempty" mapstructure:"schema_registry_replication_factor,omitempty" json:"schemaRegistryReplicationFactor,omitempty"` + SchemaRegistryAPI []NamedSocketAddress `yaml:"schema_registry_api,omitempty" json:"schemaRegistryApi,omitempty"` + SchemaRegistryAPITLS []ServerTLS `yaml:"schema_registry_api_tls,omitempty" json:"schemaRegistryApiTls,omitempty"` + SchemaRegistryReplicationFactor *int `yaml:"schema_registry_replication_factor,omitempty" json:"schemaRegistryReplicationFactor,omitempty"` } type KafkaClient struct { - Brokers []SocketAddress `yaml:"brokers,omitempty" mapstructure:"brokers,omitempty" json:"brokers,omitempty"` - BrokerTLS ServerTLS `yaml:"broker_tls,omitempty" mapstructure:"broker_tls,omitempty" json:"brokerTls,omitempty"` - SASLMechanism *string `yaml:"sasl_mechanism,omitempty" mapstructure:"sasl_mechanism,omitempty" json:"saslMechanism,omitempty"` - SCRAMUsername *string `yaml:"scram_username,omitempty" mapstructure:"scram_username,omitempty" json:"scramUsername,omitempty"` - SCRAMPassword *string `yaml:"scram_password,omitempty" mapstructure:"scram_password,omitempty" json:"scramPassword,omitempty"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Brokers []SocketAddress `yaml:"brokers,omitempty" json:"brokers,omitempty"` + BrokerTLS ServerTLS `yaml:"broker_tls,omitempty" json:"brokerTls,omitempty"` + SASLMechanism *string `yaml:"sasl_mechanism,omitempty" json:"saslMechanism,omitempty"` + SCRAMUsername *string `yaml:"scram_username,omitempty" json:"scramUsername,omitempty"` + SCRAMPassword *string `yaml:"scram_password,omitempty" json:"scramPassword,omitempty"` + Other map[string]interface{} `yaml:",inline"` } type SeedServer struct { - Host SocketAddress `yaml:"host" mapstructure:"host" json:"host"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` } type SocketAddress struct { - Address string `yaml:"address" mapstructure:"address" json:"address"` - Port int `yaml:"port" mapstructure:"port" json:"port"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` } type NamedSocketAddress struct { - Address string `yaml:"address" mapstructure:"address" json:"address"` - Port int `yaml:"port" mapstructure:"port" json:"port"` - Name string `yaml:"name,omitempty" mapstructure:"name,omitempty" json:"name,omitempty"` + Address string `yaml:"address" json:"address"` + Port int `yaml:"port" json:"port"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` } type TLS struct { - KeyFile string `yaml:"key_file,omitempty" mapstructure:"key_file,omitempty" json:"keyFile"` - CertFile string `yaml:"cert_file,omitempty" mapstructure:"cert_file,omitempty" json:"certFile"` - TruststoreFile string `yaml:"truststore_file,omitempty" mapstructure:"truststore_file,omitempty" json:"truststoreFile"` + KeyFile string `yaml:"key_file,omitempty" json:"keyFile"` + CertFile string `yaml:"cert_file,omitempty" json:"certFile"` + TruststoreFile string `yaml:"truststore_file,omitempty" json:"truststoreFile"` } func (t *TLS) Config(fs afero.Fs) (*tls.Config, error) { @@ -129,62 +130,62 @@ func (t *TLS) Config(fs afero.Fs) (*tls.Config, error) { } type ServerTLS struct { - Name string `yaml:"name,omitempty" mapstructure:"name,omitempty" json:"name"` - KeyFile string `yaml:"key_file,omitempty" mapstructure:"key_file,omitempty" json:"keyFile"` - CertFile string `yaml:"cert_file,omitempty" mapstructure:"cert_file,omitempty" json:"certFile"` - TruststoreFile string `yaml:"truststore_file,omitempty" mapstructure:"truststore_file,omitempty" json:"truststoreFile"` - Enabled bool `yaml:"enabled,omitempty" mapstructure:"enabled,omitempty" json:"enabled"` - RequireClientAuth bool `yaml:"require_client_auth,omitempty" mapstructure:"require_client_auth,omitempty" json:"requireClientAuth"` - Other map[string]interface{} `yaml:",inline" mapstructure:",remain"` + Name string `yaml:"name,omitempty" json:"name"` + KeyFile string `yaml:"key_file,omitempty" json:"keyFile"` + CertFile string `yaml:"cert_file,omitempty" json:"certFile"` + TruststoreFile string `yaml:"truststore_file,omitempty" json:"truststoreFile"` + Enabled bool `yaml:"enabled,omitempty" json:"enabled"` + RequireClientAuth bool `yaml:"require_client_auth,omitempty" json:"requireClientAuth"` + Other map[string]interface{} `yaml:",inline" ` } type RpkConfig struct { // Deprecated 2021-07-1 - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` // Deprecated 2021-07-1 - SASL *SASL `yaml:"sasl,omitempty" mapstructure:"sasl,omitempty" json:"sasl,omitempty"` - - KafkaAPI RpkKafkaAPI `yaml:"kafka_api,omitempty" mapstructure:"kafka_api,omitempty" json:"kafkaApi"` - AdminAPI RpkAdminAPI `yaml:"admin_api,omitempty" mapstructure:"admin_api,omitempty" json:"adminApi"` - AdditionalStartFlags []string `yaml:"additional_start_flags,omitempty" mapstructure:"additional_start_flags,omitempty" json:"additionalStartFlags"` - EnableUsageStats bool `yaml:"enable_usage_stats" mapstructure:"enable_usage_stats" json:"enableUsageStats"` - TuneNetwork bool `yaml:"tune_network" mapstructure:"tune_network" json:"tuneNetwork"` - TuneDiskScheduler bool `yaml:"tune_disk_scheduler" mapstructure:"tune_disk_scheduler" json:"tuneDiskScheduler"` - TuneNomerges bool `yaml:"tune_disk_nomerges" mapstructure:"tune_disk_nomerges" json:"tuneNomerges"` - TuneDiskWriteCache bool `yaml:"tune_disk_write_cache" mapstructure:"tune_disk_write_cache" json:"tuneDiskWriteCache"` - TuneDiskIrq bool `yaml:"tune_disk_irq" mapstructure:"tune_disk_irq" json:"tuneDiskIrq"` - TuneFstrim bool `yaml:"tune_fstrim" mapstructure:"tune_fstrim" json:"tuneFstrim"` - TuneCPU bool `yaml:"tune_cpu" mapstructure:"tune_cpu" json:"tuneCpu"` - TuneAioEvents bool `yaml:"tune_aio_events" mapstructure:"tune_aio_events" json:"tuneAioEvents"` - TuneClocksource bool `yaml:"tune_clocksource" mapstructure:"tune_clocksource" json:"tuneClocksource"` - TuneSwappiness bool `yaml:"tune_swappiness" mapstructure:"tune_swappiness" json:"tuneSwappiness"` - TuneTransparentHugePages bool `yaml:"tune_transparent_hugepages" mapstructure:"tune_transparent_hugepages" json:"tuneTransparentHugePages"` - EnableMemoryLocking bool `yaml:"enable_memory_locking" mapstructure:"enable_memory_locking" json:"enableMemoryLocking"` - TuneCoredump bool `yaml:"tune_coredump" mapstructure:"tune_coredump" json:"tuneCoredump"` - CoredumpDir string `yaml:"coredump_dir,omitempty" mapstructure:"coredump_dir,omitempty" json:"coredumpDir"` - TuneBallastFile bool `yaml:"tune_ballast_file" mapstructure:"tune_ballast_file" json:"tuneBallastFile"` - BallastFilePath string `yaml:"ballast_file_path,omitempty" mapstructure:"ballast_file_path,omitempty" json:"ballastFilePath"` - BallastFileSize string `yaml:"ballast_file_size,omitempty" mapstructure:"ballast_file_size,omitempty" json:"ballastFileSize"` - WellKnownIo string `yaml:"well_known_io,omitempty" mapstructure:"well_known_io,omitempty" json:"wellKnownIo"` - Overprovisioned bool `yaml:"overprovisioned" mapstructure:"overprovisioned" json:"overprovisioned"` - SMP *int `yaml:"smp,omitempty" mapstructure:"smp,omitempty" json:"smp,omitempty"` + SASL *SASL `yaml:"sasl,omitempty" json:"sasl,omitempty"` + + KafkaAPI RpkKafkaAPI `yaml:"kafka_api,omitempty" json:"kafkaApi"` + AdminAPI RpkAdminAPI `yaml:"admin_api,omitempty" json:"adminApi"` + AdditionalStartFlags []string `yaml:"additional_start_flags,omitempty" json:"additionalStartFlags"` + EnableUsageStats bool `yaml:"enable_usage_stats" json:"enableUsageStats"` + TuneNetwork bool `yaml:"tune_network" json:"tuneNetwork"` + TuneDiskScheduler bool `yaml:"tune_disk_scheduler" json:"tuneDiskScheduler"` + TuneNomerges bool `yaml:"tune_disk_nomerges" json:"tuneNomerges"` + TuneDiskWriteCache bool `yaml:"tune_disk_write_cache" json:"tuneDiskWriteCache"` + TuneDiskIrq bool `yaml:"tune_disk_irq" json:"tuneDiskIrq"` + TuneFstrim bool `yaml:"tune_fstrim" json:"tuneFstrim"` + TuneCPU bool `yaml:"tune_cpu" json:"tuneCpu"` + TuneAioEvents bool `yaml:"tune_aio_events" json:"tuneAioEvents"` + TuneClocksource bool `yaml:"tune_clocksource" json:"tuneClocksource"` + TuneSwappiness bool `yaml:"tune_swappiness" json:"tuneSwappiness"` + TuneTransparentHugePages bool `yaml:"tune_transparent_hugepages" json:"tuneTransparentHugePages"` + EnableMemoryLocking bool `yaml:"enable_memory_locking" json:"enableMemoryLocking"` + TuneCoredump bool `yaml:"tune_coredump" json:"tuneCoredump"` + CoredumpDir string `yaml:"coredump_dir,omitempty" json:"coredumpDir"` + TuneBallastFile bool `yaml:"tune_ballast_file" json:"tuneBallastFile"` + BallastFilePath string `yaml:"ballast_file_path,omitempty" json:"ballastFilePath"` + BallastFileSize string `yaml:"ballast_file_size,omitempty" json:"ballastFileSize"` + WellKnownIo string `yaml:"well_known_io,omitempty" json:"wellKnownIo"` + Overprovisioned bool `yaml:"overprovisioned" json:"overprovisioned"` + SMP *int `yaml:"smp,omitempty" json:"smp,omitempty"` } type RpkKafkaAPI struct { - Brokers []string `yaml:"brokers,omitempty" mapstructure:"brokers,omitempty" json:"brokers"` - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` - SASL *SASL `yaml:"sasl,omitempty" mapstructure:"sasl,omitempty" json:"sasl,omitempty"` + Brokers []string `yaml:"brokers,omitempty" json:"brokers"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` + SASL *SASL `yaml:"sasl,omitempty" json:"sasl,omitempty"` } type RpkAdminAPI struct { - Addresses []string `yaml:"addresses,omitempty" mapstructure:"addresses,omitempty" json:"addresses"` - TLS *TLS `yaml:"tls,omitempty" mapstructure:"tls,omitempty" json:"tls"` + Addresses []string `yaml:"addresses,omitempty" json:"addresses"` + TLS *TLS `yaml:"tls,omitempty" json:"tls"` } type SASL struct { - User string `yaml:"user,omitempty" mapstructure:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" mapstructure:"password,omitempty" json:"password,omitempty"` - Mechanism string `yaml:"type,omitempty" mapstructure:"type,omitempty" json:"type,omitempty"` + User string `yaml:"user,omitempty" json:"user,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` + Mechanism string `yaml:"type,omitempty" json:"type,omitempty"` } func (c *Config) PIDFile() string { diff --git a/src/go/rpk/pkg/config/weak.go b/src/go/rpk/pkg/config/weak.go index 96e97d45b9ad6..08063f72f7b2e 100644 --- a/src/go/rpk/pkg/config/weak.go +++ b/src/go/rpk/pkg/config/weak.go @@ -246,7 +246,8 @@ func (s *serverTLSArray) UnmarshalYAML(n *yaml.Node) error { if err != nil { return err } - fmt.Fprintf(os.Stderr, "redpanda.yaml: %+v must be an array; a non-array value in array fields is deprecated and will be removed in the future\n", single) + // do not log serverTLS because the Other field may contain a secret + fmt.Fprintf(os.Stderr, "redpanda.yaml: Server TLS must be an array; a non-array value in array fields is deprecated and will be removed in the future\n") *s = []ServerTLS{single} return nil } @@ -333,6 +334,7 @@ func (rpc *RedpandaConfig) UnmarshalYAML(n *yaml.Node) error { DeveloperMode weakBool `yaml:"developer_mode"` Other map[string]interface{} `yaml:",inline"` } + if err := n.Decode(&internal); err != nil { return err } @@ -533,6 +535,30 @@ func (s *ServerTLS) UnmarshalYAML(n *yaml.Node) error { return nil } +func (ss *SeedServer) UnmarshalYAML(n *yaml.Node) error { + var internal struct { + Address weakString `yaml:"address"` + Port weakInt `yaml:"port"` + // deprecated + Host *SocketAddress `yaml:"host"` + NodeID *weakInt `yaml:"node_id"` + } + if err := n.Decode(&internal); err != nil { + return err + } + if internal.NodeID != nil { + fmt.Println("redpanda yaml: redpanda.seed_servers.node_id is deprecated and unused") + } + if internal.Host != nil { + internal.Address = weakString(internal.Host.Address) + internal.Port = weakInt(internal.Host.Port) + fmt.Println("redpanda yaml: redpanda.seed_servers.host is deprecated and will be removed in the future, use redpanda.seed_servers.{address,port}") + } + ss.Address = string(internal.Address) + ss.Port = int(internal.Port) + return nil +} + func (sa *SocketAddress) UnmarshalYAML(n *yaml.Node) error { var internal struct { Address weakString `yaml:"address"` diff --git a/src/go/rpk/pkg/config/weak_test.go b/src/go/rpk/pkg/config/weak_test.go index 3f578868a8635..62ca4f08b0942 100644 --- a/src/go/rpk/pkg/config/weak_test.go +++ b/src/go/rpk/pkg/config/weak_test.go @@ -545,7 +545,7 @@ func TestSeedServers(t *testing.T) { port: 80 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, + {"0.0.0.1", 80}, }, }, { @@ -559,8 +559,8 @@ func TestSeedServers(t *testing.T) { port: 90 `, exp: []SeedServer{ - {Host: SocketAddress{"0.0.0.1", 80}}, - {Host: SocketAddress{"0.0.0.2", 90}}, + {"0.0.0.1", 80}, + {"0.0.0.2", 90}, }, }, { @@ -647,9 +647,8 @@ redpanda: name: external port: 9093 seed_servers: - - host: - address: 192.168.0.1 - port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -761,7 +760,7 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {"192.168.0.1", 33145}, }, Other: map[string]interface{}{ "enable_admin_api": true, @@ -876,9 +875,15 @@ redpanda: name: external port: 9093 seed_servers: + - host: + address: 192.168.0.1 + port: 33145 + - node_id: "0" host: address: 192.168.0.1 port: 33145 + - address: 192.168.0.1 + port: 33145 rack: "rack-id" pandaproxy: pandaproxy_api: @@ -989,7 +994,9 @@ rpk: {"redpanda-0.my.domain.com.", 9093, "external"}, }, SeedServers: []SeedServer{ - {Host: SocketAddress{"192.168.0.1", 33145}}, + {"192.168.0.1", 33145}, + {"192.168.0.1", 33145}, + {"192.168.0.1", 33145}, }, Other: map[string]interface{}{ "enable_admin_api": true, diff --git a/src/go/rpk/pkg/plugin/manifest.go b/src/go/rpk/pkg/plugin/manifest.go index e631ccbb07fdb..ecca50478f13b 100644 --- a/src/go/rpk/pkg/plugin/manifest.go +++ b/src/go/rpk/pkg/plugin/manifest.go @@ -15,7 +15,7 @@ import ( "strings" "time" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Manifest represents a plugin manifest, which is essentially a list of diff --git a/src/go/rpk/pkg/tuners/iotune/data.go b/src/go/rpk/pkg/tuners/iotune/data.go index e0af774aa69c1..3c1b631eab18c 100644 --- a/src/go/rpk/pkg/tuners/iotune/data.go +++ b/src/go/rpk/pkg/tuners/iotune/data.go @@ -14,7 +14,7 @@ import ( "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cloud/vendor" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type IoProperties struct { diff --git a/src/go/rpk/pkg/tuners/redpanda_checkers.go b/src/go/rpk/pkg/tuners/redpanda_checkers.go index 68d40ffa300ba..340b5993b9420 100644 --- a/src/go/rpk/pkg/tuners/redpanda_checkers.go +++ b/src/go/rpk/pkg/tuners/redpanda_checkers.go @@ -69,7 +69,7 @@ func NewConfigChecker(conf *config.Config) Checker { Fatal, true, func() (interface{}, error) { - ok, _ := config.Check(conf) + ok, _ := conf.Check() return ok, nil }) } diff --git a/src/go/rpk/pkg/yaml/yaml.go b/src/go/rpk/pkg/yaml/yaml.go deleted file mode 100644 index 47385a64312cb..0000000000000 --- a/src/go/rpk/pkg/yaml/yaml.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package yaml - -import ( - "github.com/spf13/afero" - "gopkg.in/yaml.v2" -) - -func Persist(fs afero.Fs, in interface{}, file string) error { - bytes, err := yaml.Marshal(in) - if err != nil { - return err - } - err = afero.WriteFile(fs, file, bytes, 0o644) - return err -} - -func Read(fs afero.Fs, out interface{}, file string) error { - content, err := afero.ReadFile(fs, file) - if err != nil { - return err - } - err = yaml.Unmarshal(content, out) - if err != nil { - return err - } - return nil -} diff --git a/tests/rptest/tests/rpk_config_test.py b/tests/rptest/tests/rpk_config_test.py index 04ab14608629d..2a4c5a9fe46bc 100644 --- a/tests/rptest/tests/rpk_config_test.py +++ b/tests/rptest/tests/rpk_config_test.py @@ -35,41 +35,47 @@ def test_config_init(self): with tempfile.TemporaryDirectory() as d: node.account.copy_from(path, d) with open(os.path.join(d, path)) as f: - expected_out = '''config_file: /root/redpanda-test.yaml + expected_out = '''config_file: ./redpanda-test.yaml # node_uuid: (the uuid is random so we don't compare it) pandaproxy: {} redpanda: - admin: - - address: 0.0.0.0 - port: 9644 - data_directory: /var/lib/redpanda/data - developer_mode: true - kafka_api: - - address: 0.0.0.0 - port: 9092 - node_id: 0 - rpc_server: - address: 0.0.0.0 - port: 33145 - seed_servers: [] + data_directory: /var/lib/redpanda/data + node_id: 0 + seed_servers: [] + rpc_server: + address: 0.0.0.0 + port: 33145 + kafka_api: + - address: 0.0.0.0 + port: 9092 + admin: + - address: 0.0.0.0 + port: 9644 + developer_mode: true rpk: - coredump_dir: /var/lib/redpanda/coredump - enable_memory_locking: false - enable_usage_stats: false - overprovisioned: false - tune_aio_events: false - tune_ballast_file: false - tune_clocksource: false - tune_coredump: false - tune_cpu: false - tune_disk_irq: false - tune_disk_nomerges: false - tune_disk_scheduler: false - tune_disk_write_cache: false - tune_fstrim: false - tune_network: false - tune_swappiness: false - tune_transparent_hugepages: false + admin_api: + addresses: + - 127.0.0.1:9644 + coredump_dir: /var/lib/redpanda/coredump + enable_memory_locking: false + enable_usage_stats: false + kafka_api: + brokers: + - 0.0.0.0:9092 + overprovisioned: false + tune_aio_events: false + tune_ballast_file: false + tune_clocksource: false + tune_coredump: false + tune_cpu: false + tune_disk_irq: false + tune_disk_nomerges: false + tune_disk_scheduler: false + tune_disk_write_cache: false + tune_fstrim: false + tune_network: false + tune_swappiness: false + tune_transparent_hugepages: false schema_registry: {} ''' @@ -97,7 +103,7 @@ def test_config_set_single_number(self): rpk = RpkRemoteTool(self.redpanda, node) config_file = 'redpanda.yaml' key = 'redpanda.admin.port' - value = '9641' # The default is 9644, so we will change it + value = 9641 # The default is 9644, so we will change it rpk.config_set(key, value, path=RedpandaService.NODE_CONFIG_FILE) @@ -106,7 +112,7 @@ def test_config_set_single_number(self): with open(os.path.join(d, config_file)) as f: actual_config = yaml.full_load(f.read()) - assert f"{actual_config['redpanda']['admin']['port']}" == value + assert f"{actual_config['redpanda']['admin'][0]['port']}" == value @cluster(num_nodes=3) def test_config_set_yaml(self): @@ -129,14 +135,31 @@ def test_config_set_yaml(self): address: 192.168.10.3 port: 33145 ''' + + expected = ''' +- address: 192.168.10.1 + port: 33145 +- address: 192.168.10.2 + port: 33145 +- address: 192.168.10.3 + port: 33145 +''' + rpk.config_set(key, value, format='yaml') with tempfile.TemporaryDirectory() as d: node.account.copy_from(RedpandaService.NODE_CONFIG_FILE, d) with open(os.path.join(d, 'redpanda.yaml')) as f: - expected_config = yaml.full_load(value) + expected_config = yaml.full_load(expected) actual_config = yaml.full_load(f.read()) + if actual_config['redpanda']['seed_servers'] != expected_config: + self.logger.error("Configs differ") + self.logger.error( + f"Expected: {yaml.dump(expected_config)}") + self.logger.error( + f"Actual: {yaml.dump(actual_config['redpanda']['seed_servers'])}" + ) assert actual_config['redpanda'][ 'seed_servers'] == expected_config @@ -146,24 +169,34 @@ def test_config_set_json(self): node = self.redpanda.get_node(n) rpk = RpkRemoteTool(self.redpanda, node) key = 'rpk' - value = '{"tune_aio_events":true,"tune_cpu":true,"tune_disk_irq":true}' + value = '{"tuneAioEvents":true,"tuneCpu":true,"tuneDiskIrq":true}' rpk.config_set(key, value, format='json') expected_config = yaml.full_load(''' +admin_api: + addresses: + - 127.0.0.1:9644 coredump_dir: /var/lib/redpanda/coredump enable_memory_locking: false enable_usage_stats: false +kafka_api: + brokers: + - 0.0.0.0:9092 +overprovisioned: false tune_aio_events: true +tune_ballast_file: false tune_clocksource: false tune_coredump: false tune_cpu: true tune_disk_irq: true tune_disk_nomerges: false tune_disk_scheduler: false +tune_disk_write_cache: false tune_fstrim: false tune_network: false tune_swappiness: false +tune_transparent_hugepages: false ''') with tempfile.TemporaryDirectory() as d: @@ -172,6 +205,12 @@ def test_config_set_json(self): with open(os.path.join(d, 'redpanda.yaml')) as f: actual_config = yaml.full_load(f.read()) + if actual_config['rpk'] != expected_config: + self.logger.error("Configs differ") + self.logger.error( + f"Expected: {yaml.dump(expected_config)}") + self.logger.error( + f"Actual: {yaml.dump(actual_config['rpk'])}") assert actual_config['rpk'] == expected_config @cluster(num_nodes=3, log_allow_list=RESTART_LOG_ALLOW_LIST)