Skip to content

Commit

Permalink
Merge pull request #6004 from vbotbuildovich/backport-fixes-to-v22.2.…
Browse files Browse the repository at this point in the history 
…x-593

[v22.2.x] rpk: use redpanda.yaml instead of bootstrap file
  • Loading branch information
@twmb
twmb committed Aug 12, 2022
2 parents 0fa3254 + ad51f06 commit 5c2dcea
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 139 deletions.
2 changes: 1 addition & 1 deletion src/go/rpk/pkg/cli/cmd/container/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ func CreateNode(
AdvertiseAddresses(ip, config.DefaultProxyPort, proxyPort),
"--advertise-rpc-addr",
net.JoinHostPort(ip, strconv.Itoa(config.Default().Redpanda.RPCServer.Port)),
"--mode container",
"--mode dev-container",
}
containerConfig := container.Config{
Image: image,
Expand Down
76 changes: 15 additions & 61 deletions src/go/rpk/pkg/cli/cmd/redpanda/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ package redpanda
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -163,17 +162,13 @@ func NewStartCommand(fs afero.Fs, launcher rp.Launcher) *cobra.Command {
// configuration itself.
cfg = cfg.FileOrDefaults()
if cfg.Redpanda.DeveloperMode && len(mode) == 0 {
mode = "container"
mode = "dev-container"
}
switch mode {
case "container":
case "dev-container":
fmt.Fprintln(os.Stderr, "WARNING: This is a setup for development purposes only; in this mode your clusters may run unrealistically fast and data can be corrupted any time your computer shuts down uncleanly.")
cfg.Redpanda.DeveloperMode = true
setContainerModeFlags(cmd)
err := setClusterProperties(fs, cfg.FileLocation())
if err != nil {
fmt.Fprintf(os.Stderr, "unable to set cluster properties: %v\n", err)
}
setContainerModeCfgFields(cfg)
case "help":
fmt.Println(helpMode)
return nil
Expand Down Expand Up @@ -1076,7 +1071,7 @@ func mergeMaps(a, b map[string]string) map[string]string {
return a
}

// setContainerModeFlags sets flags bundled into --mode container flag.
// setContainerModeFlags sets flags bundled into --mode dev-container flag.
func setContainerModeFlags(cmd *cobra.Command) {
devMap := map[string]string{
overprovisionedFlag: "true",
Expand All @@ -1085,7 +1080,7 @@ func setContainerModeFlags(cmd *cobra.Command) {
unsafeBypassFsyncFlag: "true",
}
// We don't override the values set during command execution, e.g:
// rpk redpanda start --mode container --smp 2
// rpk redpanda start --mode dev-container --smp 2
// will apply all dev flags, but smp will be 2.
for k, v := range devMap {
if !cmd.Flags().Changed(k) {
Expand All @@ -1094,64 +1089,23 @@ func setContainerModeFlags(cmd *cobra.Command) {
}
}

// setClusterProperties generates a .bootstrap.yaml file in the same path as
// the redpanda.yaml, this will be picked up by redpanda on first start and
// set the passed properties.
func setClusterProperties(fs afero.Fs, cfgFileLocation string) (rerr error) {
const props = `auto_create_topics_enabled: true
group_topic_partitions: 3
storage_min_free_bytes: 10485760
topic_partitions_per_shard: 1000
`
cfgDir := filepath.Dir(cfgFileLocation)

tmp, err := afero.TempFile(fs, cfgDir, "bootstrap-*.yaml")
if err != nil {
return err
}

defer func() {
if rerr != nil {
suggestion := "you can run 'rpk cluster config set <key> <value>' to set the following properties: " + props
if removeErr := fs.Remove(tmp.Name()); removeErr != nil {
rerr = fmt.Errorf("%s, unable to remove temp file: %v; %s", rerr, removeErr, suggestion)
} else {
rerr = fmt.Errorf("%s, temp file removed from disk; %s", rerr, suggestion)
}
}
}()

_, err = io.WriteString(tmp, props)
tmp.Close()
if err != nil {
return fmt.Errorf("error writing to temporary file: %v", err)
}

// If we already have a redpanda.yaml we want to have the same file
// ownership for .boostrap.yaml
if exists, _ := afero.Exists(fs, cfgFileLocation); exists {
stat, err := fs.Stat(cfgFileLocation)
if err != nil {
return fmt.Errorf("unable to stat existing file: %v", err)
}
err = config.PreserveUnixOwnership(fs, stat, tmp.Name())
if err != nil {
return err
}
}
func setContainerModeCfgFields(cfg *config.Config) {
cfg.Redpanda.DeveloperMode = true

err = fs.Rename(tmp.Name(), filepath.Join(cfgDir, ".bootstrap.yaml"))
if err != nil {
return err
// cluster properties:
if cfg.Redpanda.Other == nil {
cfg.Redpanda.Other = make(map[string]interface{})
}

return nil
cfg.Redpanda.Other["auto_create_topics_enabled"] = true
cfg.Redpanda.Other["group_topic_partitions"] = 3
cfg.Redpanda.Other["storage_min_free_bytes"] = 10485760
cfg.Redpanda.Other["topic_partitions_per_shard"] = 1000
}

const helpMode = `Mode uses well-known configuration properties for development or tests
environments:
--mode container
--mode dev-container
Bundled flags:
* --overprovisioned
* --reserve-memory 0M
Expand Down
145 changes: 68 additions & 77 deletions src/go/rpk/pkg/cli/cmd/redpanda/start_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,14 @@ func TestStartCommand(t *testing.T) {
path,
)
c := config.Default()
// We are adding now this cluster properties as default with
// redpanda.developer_mode: true.
c.Redpanda.Other = map[string]interface{}{
"auto_create_topics_enabled": true,
"group_topic_partitions": 3,
"storage_min_free_bytes": 10485760,
"topic_partitions_per_shard": 1000,
}

conf, err := new(config.Params).Load(fs)
require.NoError(st, err)
Expand Down Expand Up @@ -1374,29 +1382,39 @@ func TestStartCommand(t *testing.T) {
require.Equal(st, expected, rpArgs.ExtraArgs)
},
}, {
name: "--mode container flag set required bundle of flags",
name: "--mode dev-container flag set required bundle of flags",
args: []string{
"--install-dir", "/var/lib/redpanda",
"--mode", "container",
"--mode", "dev-container",
},
postCheck: func(
fs afero.Fs,
rpArgs *redpanda.RedpandaArgs,
st *testing.T,
) {
// Flags:
require.Equal(st, "true", rpArgs.SeastarFlags["overprovisioned"])
require.Equal(st, "0M", rpArgs.SeastarFlags["reserve-memory"])
require.Equal(st, "true", rpArgs.SeastarFlags["unsafe-bypass-fsync"])

// Config:
conf, err := new(config.Params).Load(fs)
require.NoError(st, err)
require.Equal(st, 0, conf.Redpanda.ID)
require.Equal(st, true, conf.Redpanda.DeveloperMode)
expectedClusterFields := map[string]interface{}{
"auto_create_topics_enabled": true,
"group_topic_partitions": 3,
"storage_min_free_bytes": 10485760,
"topic_partitions_per_shard": 1000,
}
require.Equal(st, expectedClusterFields, conf.Redpanda.Other)
},
}, {
name: "override values set by --mode container",
name: "override flags set by --mode dev-container",
args: []string{
"--install-dir", "/var/lib/redpanda",
"--mode", "container", "--reserve-memory", "2M",
"--mode", "dev-container", "--reserve-memory", "2M",
},
postCheck: func(
fs afero.Fs,
Expand All @@ -1405,7 +1423,7 @@ func TestStartCommand(t *testing.T) {
) {
// override value:
require.Equal(st, "2M", rpArgs.SeastarFlags["reserve-memory"])
// rest of --mode container bundle
// rest of --mode dev-container bundle
require.Equal(st, "true", rpArgs.SeastarFlags["overprovisioned"])
require.Equal(st, "true", rpArgs.SeastarFlags["unsafe-bypass-fsync"])
conf, err := new(config.Params).Load(fs)
Expand All @@ -1414,62 +1432,7 @@ func TestStartCommand(t *testing.T) {
require.Equal(st, true, conf.Redpanda.DeveloperMode)
},
}, {
name: ".bootstrap.yaml created with --mode container",
args: []string{
"--install-dir", "/var/lib/redpanda",
"--mode", "container",
},
postCheck: func(
fs afero.Fs,
_ *redpanda.RedpandaArgs,
st *testing.T,
) {
bFile := "/etc/redpanda/.bootstrap.yaml"
exists, err := afero.Exists(fs, bFile)
require.NoError(st, err)
require.True(st, exists)
file, err := afero.ReadFile(fs, bFile)
require.NoError(st, err)
require.Equal(
st,
`auto_create_topics_enabled: true
group_topic_partitions: 3
storage_min_free_bytes: 10485760
topic_partitions_per_shard: 1000
`,
string(file),
)
},
}, {
name: ".bootstrap.yaml created with --mode container in arbitrary path",
args: []string{
"--install-dir", "/var/lib/redpanda",
"--mode", "container", "--config",
"/arbitrary/path/redpanda.yaml",
},
postCheck: func(
fs afero.Fs,
_ *redpanda.RedpandaArgs,
st *testing.T,
) {
bFile := "/arbitrary/path/.bootstrap.yaml"
exists, err := afero.Exists(fs, bFile)
require.NoError(st, err)
require.True(st, exists)
file, err := afero.ReadFile(fs, bFile)
require.NoError(st, err)
require.Equal(
st,
`auto_create_topics_enabled: true
group_topic_partitions: 3
storage_min_free_bytes: 10485760
topic_partitions_per_shard: 1000
`,
string(file),
)
},
}, {
name: "redpanda.developer_mode: true behaves like --mode container",
name: "redpanda.developer_mode: true behaves like --mode dev-container",
args: []string{"--install-dir", "/var/lib/redpanda"},
before: func(fs afero.Fs) error {
conf, _ := new(config.Params).Load(fs)
Expand All @@ -1488,25 +1451,53 @@ topic_partitions_per_shard: 1000
require.NoError(st, err)

// Config:
expectedClusterFields := map[string]interface{}{
"auto_create_topics_enabled": true,
"group_topic_partitions": 3,
"storage_min_free_bytes": 10485760,
"topic_partitions_per_shard": 1000,
}
require.Equal(st, 0, conf.Redpanda.ID)
require.Equal(st, true, conf.Redpanda.DeveloperMode)

// Bootstrap Yaml
bFile := "/etc/redpanda/.bootstrap.yaml"
exists, err := afero.Exists(fs, bFile)
require.NoError(st, err)
require.True(st, exists)
file, err := afero.ReadFile(fs, bFile)
require.NoError(st, err)
require.Equal(
st,
`auto_create_topics_enabled: true
group_topic_partitions: 3
storage_min_free_bytes: 10485760
topic_partitions_per_shard: 1000
`,
string(file),
require.Equal(st, expectedClusterFields, conf.Redpanda.Other)
},
}, {
name: "--set overrides cluster configs set by --mode dev-container",
args: []string{
"--install-dir", "/var/lib/redpanda",
"--mode", "dev-container",
},
before: func(fs afero.Fs) error {
// --set flags are parsed "outside" of Cobra, directly from
// os.Args.
os.Args = append(
os.Args,
// A single int value
"--set", "redpanda.auto_create_topics_enabled=false",
// A single bool value
"--set", "redpanda.group_topic_partitions=1",
)
return nil
},
after: func() {
for i, a := range os.Args {
if a == setFlag {
os.Args = os.Args[:i]
return
}
}
},
postCheck: func(fs afero.Fs, _ *redpanda.RedpandaArgs, st *testing.T) {
conf, _ := new(config.Params).Load(fs)
expectedClusterFields := map[string]interface{}{
// set by --set flag
"auto_create_topics_enabled": false,
"group_topic_partitions": 1,
// rest of --mode dev-container cfg fields
"storage_min_free_bytes": 10485760,
"topic_partitions_per_shard": 1000,
}
require.Exactly(st, expectedClusterFields, conf.Redpanda.Other)
},
}, {
name: "Fails if unknown mode is passed",
Expand Down

0 comments on commit 5c2dcea

Please sign in to comment.