diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2d5be115..e8446f7cc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re - [#3700](https://github.com/thanos-io/thanos/pull/3700) ui: make old bucket viewer UI work with vanilla Prometheus blocks - [#2641](https://github.com/thanos-io/thanos/issues/2641) Query Frontend: Added `--query-range.request-downsampled` flag enabling additional queries for downsampled data in case of empty or incomplete response to range request. +### Fixed + +- [#3773](https://github.com/thanos-io/thanos/pull/3773) Compact: Pad compaction planner size check + ### Changed - [#3705](https://github.com/thanos-io/thanos/pull/3705) Store: Fix race condition leading to failing queries or possibly incorrect query results. diff --git a/pkg/compact/planner.go b/pkg/compact/planner.go index 208d4832a4..59a334905e 100644 --- a/pkg/compact/planner.go +++ b/pkg/compact/planner.go @@ -234,7 +234,7 @@ type largeTotalIndexSizeFilter struct { var _ Planner = &largeTotalIndexSizeFilter{} // WithLargeTotalIndexSizeFilter wraps Planner with largeTotalIndexSizeFilter that checks the given plans and estimates total index size. -// When found, it marks block for no compaction by placing no-compact.json and updating cache. +// When found, it marks block for no compaction by placing no-compact-mark.json and updating cache. // NOTE: The estimation is very rough as it assumes extreme cases of indexes sharing no bytes, thus summing all source index sizes. // Adjust limit accordingly reducing to some % of actual limit you want to give. // TODO(bwplotka): This is short term fix for https://github.com/thanos-io/thanos/issues/1424, replace with vertical block sharding https://github.com/thanos-io/thanos/pull/3390. @@ -278,7 +278,8 @@ PlanLoop: biggestIndex = i } totalIndexBytes += indexSize - if totalIndexBytes >= t.totalMaxIndexSizeBytes { + // Leave 15% headroom for index compaction bloat. + if totalIndexBytes >= int64(float64(t.totalMaxIndexSizeBytes)*0.85) { // Marking blocks for no compact to limit size. // TODO(bwplotka): Make sure to reset cache once this is done: https://github.com/thanos-io/thanos/issues/3408 if err := block.MarkForNoCompact( diff --git a/pkg/compact/planner_test.go b/pkg/compact/planner_test.go index db528fed37..0a5024efe6 100644 --- a/pkg/compact/planner_test.go +++ b/pkg/compact/planner_test.go @@ -710,11 +710,11 @@ func TestLargeTotalIndexSizeFilter_Plan(t *testing.T) { { name: "Blocks to fill the entire parent, but with last size exceeded (should not matter and not even marked).", metas: []*metadata.Meta{ - {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}}, BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, - {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}}, BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, - {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}}, BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}},