From b8198abc3e9d9a67fa50b0ae48d627da4f96d361 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 17 Dec 2021 19:56:10 +0530 Subject: [PATCH 1/6] draft implementation - exclude delete flag Signed-off-by: metonymic-smokey --- cmd/thanos/tools_bucket.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index b605b03b24..b6ab331897 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -107,7 +107,8 @@ type bucketVerifyConfig struct { } type bucketLsConfig struct { - output string + output string + excludeDelete bool } type bucketWebConfig struct { @@ -168,6 +169,8 @@ func (tbc *bucketVerifyConfig) registerBucketVerifyFlag(cmd extkingpin.FlagClaus func (tbc *bucketLsConfig) registerBucketLsFlag(cmd extkingpin.FlagClause) *bucketLsConfig { cmd.Flag("output", "Optional format in which to print each block's information. Options are 'json', 'wide' or a custom template."). Short('o').Default("").StringVar(&tbc.output) + cmd.Flag("exclude-delete", "Exclude blocks marked for deletion."). + Default("false").BoolVar(&tbc.excludeDelete) return tbc } @@ -377,6 +380,9 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } + // Temporary placeholder values - should I add deleteDelay as a parameter for Ls too? + f := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 1) + fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil, nil) if err != nil { return err @@ -440,6 +446,15 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } + if tbc.excludeDelete { + // Created fetcherMetrics since Filter required an *extprom.TxGaugeVec. + fetcherMetrics := block.NewFetcherMetrics(reg, nil, nil) + err = f.Filter(ctx, metas, fetcherMetrics.Synced) + if err != nil { + return err + } + } + for _, meta := range metas { objects++ if err := printBlock(meta); err != nil { From b069b47be9315988058b8674eef18e37f5cc9d25 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Fri, 17 Dec 2021 20:14:26 +0530 Subject: [PATCH 2/6] small refactor Signed-off-by: metonymic-smokey --- cmd/thanos/tools_bucket.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index b6ab331897..3f03eb4db8 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -380,9 +380,6 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - // Temporary placeholder values - should I add deleteDelay as a parameter for Ls too? - f := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 1) - fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil, nil) if err != nil { return err @@ -447,6 +444,9 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo } if tbc.excludeDelete { + // Temporary placeholder values - should I add deleteDelay as a parameter for Ls too? + f := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 1) + // Created fetcherMetrics since Filter required an *extprom.TxGaugeVec. fetcherMetrics := block.NewFetcherMetrics(reg, nil, nil) err = f.Filter(ctx, metas, fetcherMetrics.Synced) From 01948e5c1ebbaae76270e6a80c9280bb8d0f6d43 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 18 Dec 2021 08:40:09 +0530 Subject: [PATCH 3/6] used a more efficient approach Signed-off-by: metonymic-smokey --- cmd/thanos/tools_bucket.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 3f03eb4db8..d7e8bbf417 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -380,9 +380,16 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil, nil) - if err != nil { - return err + var fetcher *block.MetaFetcher + + if tbc.excludeDelete { + ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, block.FetcherConcurrency) + fetcher, err = block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), []block.MetadataFilter{ignoreDeletionMarkFilter}, nil) + if err != nil { + return err + } + } else { + fetcher, err = block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil, nil) } // Dummy actor to immediately kill the group after the run function returns. @@ -443,18 +450,6 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - if tbc.excludeDelete { - // Temporary placeholder values - should I add deleteDelay as a parameter for Ls too? - f := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, 1) - - // Created fetcherMetrics since Filter required an *extprom.TxGaugeVec. - fetcherMetrics := block.NewFetcherMetrics(reg, nil, nil) - err = f.Filter(ctx, metas, fetcherMetrics.Synced) - if err != nil { - return err - } - } - for _, meta := range metas { objects++ if err := printBlock(meta); err != nil { From 0ee7e7ffcec1de68f5cafbe5f92f7bdf6ba3591d Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sat, 18 Dec 2021 09:05:55 +0530 Subject: [PATCH 4/6] updated docs Signed-off-by: metonymic-smokey --- docs/components/tools.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/components/tools.md b/docs/components/tools.md index fe4f42b632..a5d33606d6 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -402,6 +402,7 @@ usage: thanos tools bucket ls [] List all blocks in the bucket. Flags: + --exclude-delete Exclude blocks marked for deletion. -h, --help Show context-sensitive help (also try --help-long and --help-man). --log.format=logfmt Log format to use. Possible options: logfmt or json. From b97ecb747820d85bac6d65771b5f8aa0d73f367c Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Sun, 19 Dec 2021 08:50:04 +0530 Subject: [PATCH 5/6] eliminated if else block Signed-off-by: metonymic-smokey --- cmd/thanos/tools_bucket.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index d7e8bbf417..8e979db2d4 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -380,16 +380,15 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - var fetcher *block.MetaFetcher + var filters []block.MetadataFilter if tbc.excludeDelete { ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, 0, block.FetcherConcurrency) - fetcher, err = block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), []block.MetadataFilter{ignoreDeletionMarkFilter}, nil) - if err != nil { - return err - } - } else { - fetcher, err = block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), nil, nil) + filters = append(filters, ignoreDeletionMarkFilter) + } + fetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix(extpromPrefix, reg), filters, nil) + if err != nil { + return err } // Dummy actor to immediately kill the group after the run function returns. From ade49a3095bf8540e239f54d8ec2239ca1eedbc3 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Mon, 20 Dec 2021 14:13:58 +0530 Subject: [PATCH 6/6] updated CHANGELOG Signed-off-by: metonymic-smokey --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c85fc820..da61dd9c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +- [#4970](https://github.com/thanos-io/thanos/pull/4970) Added a new flag `exclude-delete` to `tools bucket ls`, which excludes blocks marked for deletion. ### Added