From 55483abd914e45097e9c1d7633e325ca548f7e69 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Fri, 8 Jul 2022 15:44:10 +0200 Subject: [PATCH 1/6] Update faillint to v1.10.0 Signed-off-by: Filip Petkovski --- .bingo/Variables.mk | 6 +++--- .bingo/faillint.mod | 2 +- .bingo/faillint.sum | 10 ++++++++++ .bingo/variables.env | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index f41b5d52fa..dbd0fe994a 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -29,11 +29,11 @@ $(BINGO): $(BINGO_DIR)/bingo.mod @echo "(re)installing $(GOBIN)/bingo-v0.6.0" @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.6.0 "github.com/bwplotka/bingo" -FAILLINT := $(GOBIN)/faillint-v1.8.0 +FAILLINT := $(GOBIN)/faillint-v1.10.0 $(FAILLINT): $(BINGO_DIR)/faillint.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. - @echo "(re)installing $(GOBIN)/faillint-v1.8.0" - @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.8.0 "github.com/fatih/faillint" + @echo "(re)installing $(GOBIN)/faillint-v1.10.0" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.10.0 "github.com/fatih/faillint" GO_BINDATA := $(GOBIN)/go-bindata-v3.1.1+incompatible $(GO_BINDATA): $(BINGO_DIR)/go-bindata.mod diff --git a/.bingo/faillint.mod b/.bingo/faillint.mod index 8229fca657..f3357c756e 100644 --- a/.bingo/faillint.mod +++ b/.bingo/faillint.mod @@ -2,4 +2,4 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT go 1.14 -require github.com/fatih/faillint v1.8.0 +require github.com/fatih/faillint v1.10.0 diff --git a/.bingo/faillint.sum b/.bingo/faillint.sum index 8666d4a187..210ab2935b 100644 --- a/.bingo/faillint.sum +++ b/.bingo/faillint.sum @@ -2,13 +2,19 @@ dmitri.shuralyov.com/go/generated v0.0.0-20170818220700-b1254a446363 h1:o4lAkfET dmitri.shuralyov.com/go/generated v0.0.0-20170818220700-b1254a446363/go.mod h1:WG7q7swWsS2f9PYpt5DoEP/EBYWx8We5UoRltn9vJl8= github.com/fatih/faillint v1.8.0 h1:wV/mhyU+FcDtXx4RByy+H2FjrwHU9hEiFMyWPYmKqPE= github.com/fatih/faillint v1.8.0/go.mod h1:Yu1OT32SIjnX4Kn/h4/YPQOuNfuITtL3Gps70ac4lQk= +github.com/fatih/faillint v1.10.0 h1:NQ2zhSNuYp0g23/6gyCSi2IfdVIfOk/JkSzpWSDEnYQ= +github.com/fatih/faillint v1.10.0/go.mod h1:upblMxCjN4sL78nBbOHFEH9UGHTSw61M3Kj9BMS0UL0= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -16,16 +22,20 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= diff --git a/.bingo/variables.env b/.bingo/variables.env index 3076be8f06..660e297bb0 100644 --- a/.bingo/variables.env +++ b/.bingo/variables.env @@ -12,7 +12,7 @@ ALERTMANAGER="${GOBIN}/alertmanager-v0.24.0" BINGO="${GOBIN}/bingo-v0.6.0" -FAILLINT="${GOBIN}/faillint-v1.8.0" +FAILLINT="${GOBIN}/faillint-v1.10.0" GO_BINDATA="${GOBIN}/go-bindata-v3.1.1+incompatible" From 65873460720957121379119383729e723bc22a37 Mon Sep 17 00:00:00 2001 From: fpetkovski Date: Thu, 17 Mar 2022 10:54:32 +0100 Subject: [PATCH 2/6] Implement query sharding This commit implements query sharding for grouping PromQL expressions. Sharding is initiated by analyzing the PromQL and extracting grouping labels. Extracted labels are propagated down to Stores which partition the response by hashmoding all series on those labels. If a query is shardable, the partitioning and merging process will be initiated by the Query Frontend. The Query Frontend will make N distinct queries across a set of Queriers and merge the results back before presenting them to the user. Signed-off-by: Filip Petkovski --- cmd/thanos/query_frontend.go | 2 + cmd/thanos/receive.go | 5 +- cmd/thanos/sidecar.go | 5 +- cmd/thanos/store.go | 5 +- docs/components/query-frontend.md | 3 + pkg/api/query/grpc.go | 2 + pkg/api/query/querypb/query.pb.go | 234 +++++--- pkg/api/query/querypb/query.proto | 5 + pkg/api/query/v1.go | 40 +- pkg/info/infopb/rpc.pb.go | 101 ++-- pkg/info/infopb/rpc.proto | 1 + pkg/query/endpointset.go | 11 + .../test-storeset-pre-v0.8.0/storeset.go | 4 + pkg/query/querier.go | 13 +- pkg/query/querier_test.go | 12 +- pkg/query/query_test.go | 2 +- pkg/query/test.go | 4 + pkg/queryfrontend/config.go | 1 + pkg/queryfrontend/queryrange_codec.go | 45 ++ pkg/queryfrontend/request.go | 10 + pkg/queryfrontend/roundtrip.go | 16 +- pkg/queryfrontend/shard_query.go | 85 +++ pkg/querysharding/analysis.go | 140 +++++ pkg/querysharding/analyzer.go | 109 ++++ pkg/querysharding/analyzer_test.go | 193 +++++++ pkg/store/bucket.go | 21 +- pkg/store/bucket_test.go | 2 +- pkg/store/prometheus.go | 24 +- pkg/store/proxy.go | 36 +- pkg/store/proxy_test.go | 49 +- pkg/store/storepb/rpc.pb.go | 512 ++++++++++++++---- pkg/store/storepb/rpc.proto | 20 + pkg/store/storepb/shard_info.go | 104 ++++ pkg/store/storepb/shard_info_test.go | 119 ++++ pkg/store/storepb/types.proto | 2 +- pkg/store/tsdb.go | 5 + test/e2e/e2ethanos/services.go | 16 +- test/e2e/query_frontend_test.go | 79 ++- test/e2e/query_test.go | 22 +- 39 files changed, 1813 insertions(+), 246 deletions(-) create mode 100644 pkg/queryfrontend/shard_query.go create mode 100644 pkg/querysharding/analysis.go create mode 100644 pkg/querysharding/analyzer.go create mode 100644 pkg/querysharding/analyzer_test.go create mode 100644 pkg/store/storepb/shard_info.go create mode 100644 pkg/store/storepb/shard_info_test.go diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 1dafaa4110..5857a4a3d0 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -135,6 +135,8 @@ func registerQueryFrontend(app *extkingpin.App) { cmd.Flag("query-frontend.forward-header", "List of headers forwarded by the query-frontend to downstream queriers, default is empty").PlaceHolder("").StringsVar(&cfg.ForwardHeaders) + cmd.Flag("query-frontend.num-shards", "Number of queriers to use when sharding PromQL queries").IntVar(&cfg.NumShards) + cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("").EnumVar(&cfg.RequestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index b55bc68622..006591a009 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -364,8 +364,9 @@ func setupAndRunGRPCServer(g *run.Group, if isReady() { minTime, maxTime := mts.TimeRange() return &infopb.StoreInfo{ - MinTime: minTime, - MaxTime: maxTime, + MinTime: minTime, + MaxTime: maxTime, + SupportsSharding: true, } } return nil diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index c5389a3fa1..8e06dee197 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -268,8 +268,9 @@ func runSidecar( if httpProbe.IsReady() { mint, maxt := promStore.Timestamps() return &infopb.StoreInfo{ - MinTime: mint, - MaxTime: maxt, + MinTime: mint, + MaxTime: maxt, + SupportsSharding: true, } } return nil diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 08a1904393..567d1baeaf 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -393,8 +393,9 @@ func runStore( if httpProbe.IsReady() { mint, maxt := bs.TimeRange() return &infopb.StoreInfo{ - MinTime: mint, - MaxTime: maxt, + MinTime: mint, + MaxTime: maxt, + SupportsSharding: true, } } return nil diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index f17897fb33..fd66e70837 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -261,6 +261,9 @@ Flags: Log queries that are slower than the specified duration. Set to 0 to disable. Set to < 0 to enable on all queries. + --query-frontend.num-shards=QUERY-FRONTEND.NUM-SHARDS + Number of queriers to use when sharding PromQL + queries --query-frontend.org-id-header= ... Request header names used to identify the source of slow queries (repeated flag). The diff --git a/pkg/api/query/grpc.go b/pkg/api/query/grpc.go index 679cb0b831..459b94b6f7 100644 --- a/pkg/api/query/grpc.go +++ b/pkg/api/query/grpc.go @@ -78,6 +78,7 @@ func (g *GRPCAPI) Query(request *querypb.QueryRequest, server querypb.Query_Quer request.EnablePartialResponse, request.EnableQueryPushdown, false, + request.ShardInfo, ) qry, err := qe.NewInstantQuery(queryable, &promql.QueryOpts{}, request.Query, ts) if err != nil { @@ -146,6 +147,7 @@ func (g *GRPCAPI) QueryRange(request *querypb.QueryRangeRequest, srv querypb.Que request.EnablePartialResponse, request.EnableQueryPushdown, false, + request.ShardInfo, ) startTime := time.Unix(request.StartTimeSeconds, 0) diff --git a/pkg/api/query/querypb/query.pb.go b/pkg/api/query/querypb/query.pb.go index a7a9711bea..2412cbeb24 100644 --- a/pkg/api/query/querypb/query.pb.go +++ b/pkg/api/query/querypb/query.pb.go @@ -31,16 +31,17 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - TimeSeconds int64 `protobuf:"varint,2,opt,name=time_seconds,json=timeSeconds,proto3" json:"time_seconds,omitempty"` - TimeoutSeconds int64 `protobuf:"varint,3,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` - MaxResolutionSeconds int64 `protobuf:"varint,4,opt,name=max_resolution_seconds,json=maxResolutionSeconds,proto3" json:"max_resolution_seconds,omitempty"` - ReplicaLabels []string `protobuf:"bytes,5,rep,name=replica_labels,json=replicaLabels,proto3" json:"replica_labels,omitempty"` - StoreMatchers []StoreMatchers `protobuf:"bytes,6,rep,name=storeMatchers,proto3" json:"storeMatchers"` - EnableDedup bool `protobuf:"varint,7,opt,name=enableDedup,proto3" json:"enableDedup,omitempty"` - EnablePartialResponse bool `protobuf:"varint,8,opt,name=enablePartialResponse,proto3" json:"enablePartialResponse,omitempty"` - EnableQueryPushdown bool `protobuf:"varint,9,opt,name=enableQueryPushdown,proto3" json:"enableQueryPushdown,omitempty"` - SkipChunks bool `protobuf:"varint,10,opt,name=skipChunks,proto3" json:"skipChunks,omitempty"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + TimeSeconds int64 `protobuf:"varint,2,opt,name=time_seconds,json=timeSeconds,proto3" json:"time_seconds,omitempty"` + TimeoutSeconds int64 `protobuf:"varint,3,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + MaxResolutionSeconds int64 `protobuf:"varint,4,opt,name=max_resolution_seconds,json=maxResolutionSeconds,proto3" json:"max_resolution_seconds,omitempty"` + ReplicaLabels []string `protobuf:"bytes,5,rep,name=replica_labels,json=replicaLabels,proto3" json:"replica_labels,omitempty"` + StoreMatchers []StoreMatchers `protobuf:"bytes,6,rep,name=storeMatchers,proto3" json:"storeMatchers"` + EnableDedup bool `protobuf:"varint,7,opt,name=enableDedup,proto3" json:"enableDedup,omitempty"` + EnablePartialResponse bool `protobuf:"varint,8,opt,name=enablePartialResponse,proto3" json:"enablePartialResponse,omitempty"` + EnableQueryPushdown bool `protobuf:"varint,9,opt,name=enableQueryPushdown,proto3" json:"enableQueryPushdown,omitempty"` + SkipChunks bool `protobuf:"varint,10,opt,name=skipChunks,proto3" json:"skipChunks,omitempty"` + ShardInfo *storepb.ShardInfo `protobuf:"bytes,11,opt,name=shard_info,json=shardInfo,proto3" json:"shard_info,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -199,18 +200,19 @@ func (*QueryResponse) XXX_OneofWrappers() []interface{} { } type QueryRangeRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - StartTimeSeconds int64 `protobuf:"varint,2,opt,name=start_time_seconds,json=startTimeSeconds,proto3" json:"start_time_seconds,omitempty"` - EndTimeSeconds int64 `protobuf:"varint,3,opt,name=end_time_seconds,json=endTimeSeconds,proto3" json:"end_time_seconds,omitempty"` - IntervalSeconds int64 `protobuf:"varint,4,opt,name=interval_seconds,json=intervalSeconds,proto3" json:"interval_seconds,omitempty"` - TimeoutSeconds int64 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` - MaxResolutionSeconds int64 `protobuf:"varint,6,opt,name=max_resolution_seconds,json=maxResolutionSeconds,proto3" json:"max_resolution_seconds,omitempty"` - ReplicaLabels []string `protobuf:"bytes,7,rep,name=replica_labels,json=replicaLabels,proto3" json:"replica_labels,omitempty"` - StoreMatchers []StoreMatchers `protobuf:"bytes,8,rep,name=storeMatchers,proto3" json:"storeMatchers"` - EnableDedup bool `protobuf:"varint,9,opt,name=enableDedup,proto3" json:"enableDedup,omitempty"` - EnablePartialResponse bool `protobuf:"varint,10,opt,name=enablePartialResponse,proto3" json:"enablePartialResponse,omitempty"` - EnableQueryPushdown bool `protobuf:"varint,11,opt,name=enableQueryPushdown,proto3" json:"enableQueryPushdown,omitempty"` - SkipChunks bool `protobuf:"varint,12,opt,name=skipChunks,proto3" json:"skipChunks,omitempty"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + StartTimeSeconds int64 `protobuf:"varint,2,opt,name=start_time_seconds,json=startTimeSeconds,proto3" json:"start_time_seconds,omitempty"` + EndTimeSeconds int64 `protobuf:"varint,3,opt,name=end_time_seconds,json=endTimeSeconds,proto3" json:"end_time_seconds,omitempty"` + IntervalSeconds int64 `protobuf:"varint,4,opt,name=interval_seconds,json=intervalSeconds,proto3" json:"interval_seconds,omitempty"` + TimeoutSeconds int64 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + MaxResolutionSeconds int64 `protobuf:"varint,6,opt,name=max_resolution_seconds,json=maxResolutionSeconds,proto3" json:"max_resolution_seconds,omitempty"` + ReplicaLabels []string `protobuf:"bytes,7,rep,name=replica_labels,json=replicaLabels,proto3" json:"replica_labels,omitempty"` + StoreMatchers []StoreMatchers `protobuf:"bytes,8,rep,name=storeMatchers,proto3" json:"storeMatchers"` + EnableDedup bool `protobuf:"varint,9,opt,name=enableDedup,proto3" json:"enableDedup,omitempty"` + EnablePartialResponse bool `protobuf:"varint,10,opt,name=enablePartialResponse,proto3" json:"enablePartialResponse,omitempty"` + EnableQueryPushdown bool `protobuf:"varint,11,opt,name=enableQueryPushdown,proto3" json:"enableQueryPushdown,omitempty"` + SkipChunks bool `protobuf:"varint,12,opt,name=skipChunks,proto3" json:"skipChunks,omitempty"` + ShardInfo *storepb.ShardInfo `protobuf:"bytes,13,opt,name=shard_info,json=shardInfo,proto3" json:"shard_info,omitempty"` } func (m *QueryRangeRequest) Reset() { *m = QueryRangeRequest{} } @@ -342,47 +344,49 @@ func init() { func init() { proto.RegisterFile("api/query/querypb/query.proto", fileDescriptor_4b2aba43925d729f) } var fileDescriptor_4b2aba43925d729f = []byte{ - // 627 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x95, 0xcf, 0x6e, 0xd3, 0x4c, - 0x14, 0xc5, 0xed, 0x2f, 0x4d, 0x9a, 0xdc, 0x34, 0x6d, 0xbf, 0x21, 0x45, 0x6e, 0x00, 0x63, 0x22, - 0x55, 0x04, 0x09, 0x25, 0x55, 0xa9, 0xd8, 0x21, 0x41, 0x01, 0xa9, 0x8b, 0x22, 0xb5, 0x6e, 0x57, - 0x6c, 0xa2, 0x49, 0x72, 0x95, 0x58, 0x75, 0x66, 0xdc, 0x99, 0x71, 0xff, 0x88, 0x3d, 0x6b, 0xde, - 0x80, 0xf7, 0xe0, 0x09, 0xba, 0xec, 0x92, 0x15, 0x82, 0xf6, 0x45, 0x90, 0xc7, 0x76, 0xb0, 0x4b, - 0x54, 0x35, 0x20, 0xb1, 0x71, 0x3c, 0xe7, 0x9c, 0x9b, 0x3b, 0x77, 0xf2, 0x8b, 0x0d, 0x0f, 0x68, - 0xe0, 0x75, 0x8e, 0x42, 0x14, 0x67, 0xf1, 0x35, 0xe8, 0xc5, 0x9f, 0xed, 0x40, 0x70, 0xc5, 0x49, - 0x49, 0x8d, 0x28, 0xe3, 0xb2, 0x51, 0x1f, 0xf2, 0x21, 0xd7, 0x52, 0x27, 0xba, 0x8b, 0xdd, 0xc6, - 0xaa, 0x54, 0x5c, 0x60, 0x47, 0x5f, 0x83, 0x5e, 0x47, 0x9d, 0x05, 0x28, 0x13, 0xcb, 0xc9, 0x5b, - 0x81, 0xe0, 0xe3, 0x7c, 0xa2, 0xf9, 0xa5, 0x00, 0x0b, 0x7b, 0x51, 0x2b, 0x17, 0x8f, 0x42, 0x94, - 0x8a, 0xd4, 0xa1, 0xa8, 0x5b, 0x5b, 0xa6, 0x63, 0xb6, 0x2a, 0x6e, 0xbc, 0x20, 0x8f, 0x60, 0x41, - 0x79, 0x63, 0xec, 0x4a, 0xec, 0x73, 0x36, 0x90, 0xd6, 0x7f, 0x8e, 0xd9, 0x2a, 0xb8, 0xd5, 0x48, - 0xdb, 0x8f, 0x25, 0xf2, 0x18, 0x96, 0xa2, 0x25, 0x0f, 0xd5, 0x24, 0x55, 0xd0, 0xa9, 0xc5, 0x44, - 0x4e, 0x83, 0x9b, 0x70, 0x77, 0x4c, 0x4f, 0xbb, 0x02, 0x25, 0xf7, 0x43, 0xe5, 0x71, 0x36, 0xc9, - 0xcf, 0xe9, 0x7c, 0x7d, 0x4c, 0x4f, 0xdd, 0x89, 0x99, 0x56, 0xad, 0xc1, 0xa2, 0xc0, 0xc0, 0xf7, - 0xfa, 0xb4, 0xeb, 0xd3, 0x1e, 0xfa, 0xd2, 0x2a, 0x3a, 0x85, 0x56, 0xc5, 0xad, 0x25, 0xea, 0x8e, - 0x16, 0xc9, 0x2b, 0xa8, 0xe9, 0x69, 0xdf, 0x51, 0xd5, 0x1f, 0xa1, 0x90, 0x56, 0xc9, 0x29, 0xb4, - 0xaa, 0x1b, 0x2b, 0xed, 0xf8, 0x08, 0xdb, 0xfb, 0x59, 0x73, 0x6b, 0xee, 0xfc, 0xdb, 0x43, 0xc3, - 0xcd, 0x57, 0x10, 0x07, 0xaa, 0xc8, 0x68, 0xcf, 0xc7, 0x37, 0x38, 0x08, 0x03, 0x6b, 0xde, 0x31, - 0x5b, 0x65, 0x37, 0x2b, 0x91, 0x4d, 0x58, 0x89, 0x97, 0xbb, 0x54, 0x28, 0x8f, 0xfa, 0x2e, 0xca, - 0x80, 0x33, 0x89, 0x56, 0x59, 0x67, 0xa7, 0x9b, 0x64, 0x1d, 0xee, 0xc4, 0x86, 0x3e, 0xef, 0xdd, - 0x50, 0x8e, 0x06, 0xfc, 0x84, 0x59, 0x15, 0x5d, 0x33, 0xcd, 0x22, 0x36, 0x80, 0x3c, 0xf4, 0x82, - 0xd7, 0xa3, 0x90, 0x1d, 0x4a, 0x0b, 0x74, 0x30, 0xa3, 0x34, 0xf7, 0xa0, 0x96, 0x9b, 0x87, 0xbc, - 0x84, 0x9a, 0x3e, 0x9c, 0xc9, 0xf4, 0xa6, 0x9e, 0xbe, 0x9e, 0x4e, 0xbf, 0x93, 0x31, 0xd3, 0xe1, - 0x73, 0x05, 0xcd, 0x63, 0xa8, 0x25, 0x38, 0x24, 0xbb, 0xbe, 0x0f, 0xe5, 0x13, 0x2a, 0x98, 0xc7, - 0x86, 0x32, 0x46, 0x62, 0xdb, 0x70, 0x27, 0x0a, 0x79, 0x01, 0x10, 0xfd, 0xba, 0x12, 0x85, 0x87, - 0x31, 0x15, 0xd5, 0x8d, 0x7b, 0x11, 0x5a, 0x63, 0x54, 0x23, 0x0c, 0x65, 0xb7, 0xcf, 0x83, 0xb3, - 0xf6, 0x81, 0xc6, 0x24, 0x8a, 0x6c, 0x1b, 0x6e, 0xa6, 0x60, 0xab, 0x0c, 0x25, 0x81, 0x32, 0xf4, - 0x55, 0xf3, 0xf3, 0x1c, 0xfc, 0x1f, 0x37, 0xa6, 0x6c, 0x88, 0x37, 0xc3, 0xf8, 0x14, 0x88, 0x54, - 0x54, 0xa8, 0xee, 0x14, 0x24, 0x97, 0xb5, 0x73, 0x90, 0xe1, 0xb2, 0x05, 0xcb, 0xc8, 0x06, 0xf9, - 0x6c, 0x02, 0x26, 0xb2, 0x41, 0x36, 0xf9, 0x04, 0x96, 0x3d, 0xa6, 0x50, 0x1c, 0x53, 0xff, 0x1a, - 0x92, 0x4b, 0xa9, 0x7e, 0x03, 0xec, 0xc5, 0x19, 0x61, 0x2f, 0xcd, 0x04, 0xfb, 0xfc, 0xad, 0x60, - 0x2f, 0xff, 0x2d, 0xec, 0x95, 0x19, 0x60, 0x87, 0x3f, 0x80, 0xbd, 0x7a, 0x5b, 0xd8, 0x17, 0x7e, - 0x83, 0xfd, 0x03, 0x90, 0x2c, 0x20, 0xff, 0x14, 0xcf, 0x8d, 0x8f, 0x26, 0x14, 0x75, 0x77, 0xf2, - 0x3c, 0xbd, 0x99, 0xfc, 0xa9, 0xb2, 0x8f, 0xcf, 0xc6, 0xca, 0x35, 0x35, 0xde, 0xe6, 0xba, 0x49, - 0xde, 0x02, 0xfc, 0xda, 0x3e, 0x59, 0xcd, 0xc7, 0x32, 0xcc, 0x37, 0x1a, 0xd3, 0xac, 0xf4, 0x6b, - 0xb6, 0xd6, 0xce, 0x7f, 0xd8, 0xc6, 0xf9, 0xa5, 0x6d, 0x5e, 0x5c, 0xda, 0xe6, 0xf7, 0x4b, 0xdb, - 0xfc, 0x74, 0x65, 0x1b, 0x17, 0x57, 0xb6, 0xf1, 0xf5, 0xca, 0x36, 0xde, 0xcf, 0x27, 0x6f, 0x8f, - 0x5e, 0x49, 0x3f, 0xdd, 0x9f, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x4f, 0xb6, 0xe9, 0xc9, 0x59, - 0x06, 0x00, 0x00, + // 666 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0x4d, 0x6f, 0xd3, 0x4a, + 0x14, 0xb5, 0x5f, 0x9a, 0x34, 0xb9, 0x6e, 0xfa, 0x31, 0x2f, 0x7d, 0xcf, 0x0d, 0x60, 0x4c, 0xa4, + 0x0a, 0x23, 0xa1, 0xa4, 0x0a, 0x15, 0x3b, 0x24, 0x28, 0x20, 0x15, 0xa9, 0x48, 0xad, 0xdb, 0x15, + 0x9b, 0x68, 0x92, 0x4c, 0x13, 0xab, 0xce, 0x8c, 0x3b, 0x33, 0xee, 0x87, 0xd8, 0xb3, 0xe6, 0x67, + 0x75, 0x59, 0x76, 0xac, 0x10, 0xb4, 0x5b, 0x7e, 0x04, 0xf2, 0xf8, 0x03, 0xbb, 0x8a, 0x4a, 0x0b, + 0x12, 0x1b, 0x67, 0xe6, 0x9c, 0x73, 0xed, 0x3b, 0x57, 0xe7, 0x64, 0xe0, 0x1e, 0x0e, 0xbc, 0xce, + 0x61, 0x48, 0xf8, 0x69, 0xfc, 0x0c, 0xfa, 0xf1, 0x6f, 0x3b, 0xe0, 0x4c, 0x32, 0x54, 0x91, 0x63, + 0x4c, 0x99, 0x68, 0x36, 0x46, 0x6c, 0xc4, 0x14, 0xd4, 0x89, 0x56, 0x31, 0xdb, 0x5c, 0x11, 0x92, + 0x71, 0xd2, 0x51, 0xcf, 0xa0, 0xdf, 0x91, 0xa7, 0x01, 0x11, 0x09, 0xf5, 0x7f, 0x91, 0xe2, 0xc1, + 0x20, 0x21, 0xec, 0x22, 0x11, 0x70, 0x36, 0x29, 0x96, 0xb6, 0xbe, 0x97, 0x60, 0x6e, 0x27, 0xea, + 0xc1, 0x25, 0x87, 0x21, 0x11, 0x12, 0x35, 0xa0, 0xac, 0x7a, 0x32, 0x75, 0x5b, 0x77, 0x6a, 0x6e, + 0xbc, 0x41, 0x0f, 0x60, 0x4e, 0x7a, 0x13, 0xd2, 0x13, 0x64, 0xc0, 0xe8, 0x50, 0x98, 0xff, 0xd8, + 0xba, 0x53, 0x72, 0x8d, 0x08, 0xdb, 0x8d, 0x21, 0xf4, 0x10, 0x16, 0xa2, 0x2d, 0x0b, 0x65, 0xa6, + 0x2a, 0x29, 0xd5, 0x7c, 0x02, 0xa7, 0xc2, 0x75, 0xf8, 0x6f, 0x82, 0x4f, 0x7a, 0x9c, 0x08, 0xe6, + 0x87, 0xd2, 0x63, 0x34, 0xd3, 0xcf, 0x28, 0x7d, 0x63, 0x82, 0x4f, 0xdc, 0x8c, 0x4c, 0xab, 0x56, + 0x61, 0x9e, 0x93, 0xc0, 0xf7, 0x06, 0xb8, 0xe7, 0xe3, 0x3e, 0xf1, 0x85, 0x59, 0xb6, 0x4b, 0x4e, + 0xcd, 0xad, 0x27, 0xe8, 0x96, 0x02, 0xd1, 0x0b, 0xa8, 0xab, 0xd3, 0xbe, 0xc5, 0x72, 0x30, 0x26, + 0x5c, 0x98, 0x15, 0xbb, 0xe4, 0x18, 0xdd, 0xe5, 0x76, 0x3c, 0xdb, 0xf6, 0x6e, 0x9e, 0xdc, 0x98, + 0x39, 0xfb, 0x72, 0x5f, 0x73, 0x8b, 0x15, 0xc8, 0x06, 0x83, 0x50, 0xdc, 0xf7, 0xc9, 0x2b, 0x32, + 0x0c, 0x03, 0x73, 0xd6, 0xd6, 0x9d, 0xaa, 0x9b, 0x87, 0xd0, 0x3a, 0x2c, 0xc7, 0xdb, 0x6d, 0xcc, + 0xa5, 0x87, 0x7d, 0x97, 0x88, 0x80, 0x51, 0x41, 0xcc, 0xaa, 0xd2, 0x4e, 0x27, 0xd1, 0x1a, 0xfc, + 0x1b, 0x13, 0x6a, 0xde, 0xdb, 0xa1, 0x18, 0x0f, 0xd9, 0x31, 0x35, 0x6b, 0xaa, 0x66, 0x1a, 0x85, + 0x2c, 0x00, 0x71, 0xe0, 0x05, 0x2f, 0xc7, 0x21, 0x3d, 0x10, 0x26, 0x28, 0x61, 0x0e, 0x41, 0x6b, + 0x00, 0x62, 0x8c, 0xf9, 0xb0, 0xe7, 0xd1, 0x7d, 0x66, 0x1a, 0xb6, 0xee, 0x18, 0xdd, 0xa5, 0xec, + 0xa4, 0x11, 0xf3, 0x86, 0xee, 0x33, 0xb7, 0x26, 0xd2, 0x65, 0x6b, 0x07, 0xea, 0x85, 0x09, 0xa0, + 0xe7, 0x50, 0x57, 0xe3, 0xcc, 0xe6, 0xa5, 0xab, 0x79, 0x35, 0xd2, 0xb7, 0x6c, 0xe5, 0xc8, 0x74, + 0x5c, 0x85, 0x82, 0xd6, 0x11, 0xd4, 0x13, 0x03, 0x25, 0xe7, 0xbc, 0x0b, 0xd5, 0x63, 0xcc, 0xa9, + 0x47, 0x47, 0x22, 0x36, 0xd1, 0xa6, 0xe6, 0x66, 0x08, 0x7a, 0x06, 0x10, 0xf9, 0x41, 0x10, 0xee, + 0x91, 0xd8, 0x47, 0x46, 0xf7, 0x4e, 0x64, 0xc6, 0x09, 0x91, 0x63, 0x12, 0x8a, 0xde, 0x80, 0x05, + 0xa7, 0xed, 0x3d, 0x65, 0xac, 0x48, 0xb2, 0xa9, 0xb9, 0xb9, 0x82, 0x8d, 0x2a, 0x54, 0x38, 0x11, + 0xa1, 0x2f, 0x5b, 0x9f, 0x66, 0x60, 0x29, 0xfe, 0x30, 0xa6, 0x23, 0x72, 0xbd, 0x7d, 0x1f, 0x03, + 0x12, 0x12, 0x73, 0xd9, 0x9b, 0x62, 0xe2, 0x45, 0xc5, 0xec, 0xe5, 0x9c, 0xec, 0xc0, 0x22, 0xa1, + 0xc3, 0xa2, 0x36, 0xb1, 0x32, 0xa1, 0xc3, 0xbc, 0xf2, 0x11, 0x2c, 0x7a, 0x54, 0x12, 0x7e, 0x84, + 0xfd, 0x2b, 0x26, 0x5e, 0x48, 0xf1, 0x6b, 0xe2, 0x51, 0xbe, 0x65, 0x3c, 0x2a, 0xb7, 0x8a, 0xc7, + 0xec, 0x8d, 0xe2, 0x51, 0xfd, 0xd3, 0x78, 0xd4, 0x6e, 0x11, 0x0f, 0xf8, 0x8d, 0x78, 0x18, 0x37, + 0x8d, 0xc7, 0xdc, 0x2f, 0xe2, 0x51, 0xbf, 0x41, 0x3c, 0xde, 0x03, 0xca, 0x5b, 0xea, 0xaf, 0x1a, + 0xba, 0xfb, 0x41, 0x87, 0xb2, 0xfa, 0x3a, 0x7a, 0x9a, 0x2e, 0xb2, 0x18, 0xe6, 0xff, 0xa2, 0x9b, + 0xcb, 0x57, 0xd0, 0xb8, 0xcd, 0x35, 0x1d, 0xbd, 0x06, 0xf8, 0xd9, 0x3e, 0x5a, 0x29, 0xca, 0x72, + 0x29, 0x69, 0x36, 0xa7, 0x51, 0xe9, 0x6b, 0x36, 0x56, 0xcf, 0xbe, 0x59, 0xda, 0xd9, 0x85, 0xa5, + 0x9f, 0x5f, 0x58, 0xfa, 0xd7, 0x0b, 0x4b, 0xff, 0x78, 0x69, 0x69, 0xe7, 0x97, 0x96, 0xf6, 0xf9, + 0xd2, 0xd2, 0xde, 0xcd, 0x26, 0x57, 0x57, 0xbf, 0xa2, 0x6e, 0x90, 0x27, 0x3f, 0x02, 0x00, 0x00, + 0xff, 0xff, 0x8b, 0xf0, 0xf5, 0x98, 0xd6, 0x06, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -575,6 +579,18 @@ func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ShardInfo != nil { + { + size, err := m.ShardInfo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a + } if m.SkipChunks { i-- if m.SkipChunks { @@ -787,6 +803,18 @@ func (m *QueryRangeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ShardInfo != nil { + { + size, err := m.ShardInfo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x6a + } if m.SkipChunks { i-- if m.SkipChunks { @@ -1006,6 +1034,10 @@ func (m *QueryRequest) Size() (n int) { if m.SkipChunks { n += 2 } + if m.ShardInfo != nil { + l = m.ShardInfo.Size() + n += 1 + l + sovQuery(uint64(l)) + } return n } @@ -1107,6 +1139,10 @@ func (m *QueryRangeRequest) Size() (n int) { if m.SkipChunks { n += 2 } + if m.ShardInfo != nil { + l = m.ShardInfo.Size() + n += 1 + l + sovQuery(uint64(l)) + } return n } @@ -1415,6 +1451,42 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } } m.SkipChunks = bool(v != 0) + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ShardInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ShardInfo == nil { + m.ShardInfo = &storepb.ShardInfo{} + } + if err := m.ShardInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -1939,6 +2011,42 @@ func (m *QueryRangeRequest) Unmarshal(dAtA []byte) error { } } m.SkipChunks = bool(v != 0) + case 13: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ShardInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ShardInfo == nil { + m.ShardInfo = &storepb.ShardInfo{} + } + if err := m.ShardInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) diff --git a/pkg/api/query/querypb/query.proto b/pkg/api/query/querypb/query.proto index 4420b58687..6c3c589be6 100644 --- a/pkg/api/query/querypb/query.proto +++ b/pkg/api/query/querypb/query.proto @@ -8,6 +8,7 @@ option go_package = "querypb"; import "gogoproto/gogo.proto"; import "store/storepb/types.proto"; +import "store/storepb/rpc.proto"; import "store/storepb/prompb/types.proto"; option (gogoproto.sizer_all) = true; @@ -36,6 +37,8 @@ message QueryRequest { bool enablePartialResponse = 8; bool enableQueryPushdown = 9; bool skipChunks = 10; + + ShardInfo shard_info = 11; } message StoreMatchers { @@ -70,6 +73,8 @@ message QueryRangeRequest { bool enablePartialResponse = 10; bool enableQueryPushdown = 11; bool skipChunks = 12; + + ShardInfo shard_info = 13; } message QueryRangeResponse { diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 7a8fbf5fca..6ef7181f79 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -21,6 +21,7 @@ package v1 import ( "context" + "encoding/json" "math" "net/http" "sort" @@ -70,6 +71,7 @@ const ( StoreMatcherParam = "storeMatch[]" Step = "step" Stats = "stats" + ShardInfoParam = "shard_info" ) // QueryAPI is an API used by Thanos Querier. @@ -296,6 +298,24 @@ func (qapi *QueryAPI) parseStep(r *http.Request, defaultRangeQueryStep time.Dura return d, nil } +func (qapi *QueryAPI) parseShardInfo(r *http.Request) (*storepb.ShardInfo, *api.ApiError) { + data := r.FormValue(ShardInfoParam) + if data == "" { + return nil, nil + } + + if len(data) == 0 { + return nil, nil + } + + var info storepb.ShardInfo + if err := json.Unmarshal([]byte(data), &info); err != nil { + return nil, &api.ApiError{Typ: api.ErrorBadData, Err: errors.Wrapf(err, "could not unmarshal parameter %s", ShardInfoParam)} + } + + return &info, nil +} + func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiError) { ts, err := parseTimeParam(r, "time", qapi.baseAPI.Now()) if err != nil { @@ -339,13 +359,18 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro return nil, nil, apiErr } + shardInfo, apiErr := qapi.parseShardInfo(r) + if apiErr != nil { + return nil, nil, apiErr + } + qe := qapi.queryEngine(maxSourceResolution) // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_instant_query") defer span.Finish() - qry, err := qe.NewInstantQuery(qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, maxSourceResolution, enablePartialResponse, qapi.enableQueryPushdown, false), &promql.QueryOpts{}, r.FormValue("query"), ts) + qry, err := qe.NewInstantQuery(qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, maxSourceResolution, enablePartialResponse, qapi.enableQueryPushdown, false, shardInfo), &promql.QueryOpts{}, r.FormValue("query"), ts) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} } @@ -453,6 +478,11 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr } + shardInfo, apiErr := qapi.parseShardInfo(r) + if apiErr != nil { + return nil, nil, apiErr + } + qe := qapi.queryEngine(maxSourceResolution) // Record the query range requested. @@ -463,7 +493,7 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap defer span.Finish() qry, err := qe.NewRangeQuery( - qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, maxSourceResolution, enablePartialResponse, qapi.enableQueryPushdown, false), + qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, maxSourceResolution, enablePartialResponse, qapi.enableQueryPushdown, false, shardInfo), &promql.QueryOpts{}, r.FormValue("query"), start, @@ -538,7 +568,7 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A matcherSets = append(matcherSets, matchers) } - q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, qapi.enableQueryPushdown, true). + q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, qapi.enableQueryPushdown, true, nil). Querier(ctx, timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} @@ -625,7 +655,7 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, apiErr } - q, err := qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, math.MaxInt64, enablePartialResponse, qapi.enableQueryPushdown, true). + q, err := qapi.queryableCreate(enableDedup, replicaLabels, storeDebugMatchers, math.MaxInt64, enablePartialResponse, qapi.enableQueryPushdown, true, nil). Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} @@ -675,7 +705,7 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap matcherSets = append(matcherSets, matchers) } - q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, qapi.enableQueryPushdown, true). + q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, qapi.enableQueryPushdown, true, nil). Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} diff --git a/pkg/info/infopb/rpc.pb.go b/pkg/info/infopb/rpc.pb.go index 703fa1ff80..c480ce8149 100644 --- a/pkg/info/infopb/rpc.pb.go +++ b/pkg/info/infopb/rpc.pb.go @@ -117,8 +117,9 @@ var xxx_messageInfo_InfoResponse proto.InternalMessageInfo // StoreInfo holds the metadata related to Store API exposed by the component. type StoreInfo struct { - MinTime int64 `protobuf:"varint,1,opt,name=min_time,json=minTime,proto3" json:"min_time,omitempty"` - MaxTime int64 `protobuf:"varint,2,opt,name=max_time,json=maxTime,proto3" json:"max_time,omitempty"` + MinTime int64 `protobuf:"varint,1,opt,name=min_time,json=minTime,proto3" json:"min_time,omitempty"` + MaxTime int64 `protobuf:"varint,2,opt,name=max_time,json=maxTime,proto3" json:"max_time,omitempty"` + SupportsSharding bool `protobuf:"varint,3,opt,name=supports_sharding,json=supportsSharding,proto3" json:"supports_sharding,omitempty"` } func (m *StoreInfo) Reset() { *m = StoreInfo{} } @@ -355,37 +356,38 @@ func init() { func init() { proto.RegisterFile("info/infopb/rpc.proto", fileDescriptor_a1214ec45d2bf952) } var fileDescriptor_a1214ec45d2bf952 = []byte{ - // 465 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0xcf, 0x6a, 0xdb, 0x40, - 0x10, 0xc6, 0xa5, 0xf8, 0x5f, 0x34, 0x8a, 0x53, 0xba, 0xa4, 0x45, 0xf6, 0x41, 0x31, 0x22, 0x07, - 0x1f, 0x8a, 0x04, 0x2e, 0x94, 0x42, 0x4f, 0x49, 0x08, 0x34, 0xd0, 0x40, 0xab, 0xf8, 0x94, 0x8b, - 0x59, 0xa7, 0x13, 0x57, 0xa0, 0xd5, 0x6e, 0xb4, 0x6b, 0xb0, 0xdf, 0xa2, 0xaf, 0xd2, 0xb7, 0xf0, - 0x31, 0xc7, 0x9e, 0x4a, 0x6b, 0xbf, 0x48, 0xd9, 0x5d, 0x25, 0xb5, 0xa8, 0x4f, 0xb9, 0x48, 0xbb, - 0xf3, 0xfb, 0xbe, 0xd9, 0x9d, 0x61, 0x16, 0x5e, 0x65, 0xc5, 0x1d, 0x4f, 0xf4, 0x47, 0x4c, 0x93, - 0x52, 0xdc, 0xc6, 0xa2, 0xe4, 0x8a, 0x13, 0x5f, 0x7d, 0xa3, 0x05, 0x97, 0xb1, 0x06, 0xfd, 0x9e, - 0x54, 0xbc, 0xc4, 0x24, 0xa7, 0x53, 0xcc, 0xc5, 0x34, 0x51, 0x4b, 0x81, 0xd2, 0xea, 0xfa, 0x47, - 0x33, 0x3e, 0xe3, 0x66, 0x99, 0xe8, 0x95, 0x8d, 0x46, 0x5d, 0xf0, 0x2f, 0x8b, 0x3b, 0x9e, 0xe2, - 0xfd, 0x1c, 0xa5, 0x8a, 0x7e, 0x34, 0xe0, 0xc0, 0xee, 0xa5, 0xe0, 0x85, 0x44, 0xf2, 0x0e, 0xc0, - 0x24, 0x9b, 0x48, 0x54, 0x32, 0x70, 0x07, 0x8d, 0xa1, 0x3f, 0x7a, 0x19, 0x57, 0x47, 0xde, 0x7c, - 0xd2, 0xe8, 0x1a, 0xd5, 0x59, 0x73, 0xf5, 0xeb, 0xd8, 0x49, 0xbd, 0xbc, 0xda, 0x4b, 0x72, 0x02, - 0xdd, 0x73, 0xce, 0x04, 0x2f, 0xb0, 0x50, 0xe3, 0xa5, 0xc0, 0x60, 0x6f, 0xe0, 0x0e, 0xbd, 0xb4, - 0x1e, 0x24, 0x6f, 0xa0, 0x65, 0x2e, 0x1c, 0x34, 0x06, 0xee, 0xd0, 0x1f, 0xbd, 0x8e, 0xb7, 0x6a, - 0x89, 0xaf, 0x35, 0x31, 0x97, 0xb1, 0x22, 0xad, 0x2e, 0xe7, 0x39, 0xca, 0xa0, 0xb9, 0x43, 0x9d, - 0x6a, 0x62, 0xd5, 0x46, 0x44, 0x3e, 0xc2, 0x0b, 0x86, 0xaa, 0xcc, 0x6e, 0x27, 0x0c, 0x15, 0xfd, - 0x4a, 0x15, 0x0d, 0x5a, 0xc6, 0x77, 0x5c, 0xf3, 0x5d, 0x19, 0xcd, 0x55, 0x25, 0x31, 0x09, 0x0e, - 0x59, 0x2d, 0x46, 0x46, 0xd0, 0x51, 0xb4, 0x9c, 0xe9, 0x06, 0xb4, 0x4d, 0x86, 0xa0, 0x96, 0x61, - 0x6c, 0x99, 0xb1, 0x3e, 0x0a, 0xc9, 0x7b, 0xf0, 0x70, 0x81, 0x4c, 0xe4, 0xb4, 0x94, 0x41, 0xc7, - 0xb8, 0xfa, 0x35, 0xd7, 0xc5, 0x23, 0x35, 0xbe, 0x7f, 0x62, 0x92, 0x40, 0xeb, 0x7e, 0x8e, 0xe5, - 0x32, 0xd8, 0x37, 0xae, 0x5e, 0xcd, 0xf5, 0x45, 0x93, 0xd3, 0xcf, 0x97, 0xb6, 0x50, 0xa3, 0x8b, - 0x4e, 0xc1, 0x7b, 0x6a, 0x15, 0xe9, 0xc1, 0x3e, 0xcb, 0x8a, 0x89, 0xca, 0x18, 0x06, 0xee, 0xc0, - 0x1d, 0x36, 0xd2, 0x0e, 0xcb, 0x8a, 0x71, 0xc6, 0xd0, 0x20, 0xba, 0xb0, 0x68, 0xaf, 0x42, 0x74, - 0xa1, 0x51, 0xe4, 0x83, 0xf7, 0xd4, 0xbf, 0xe8, 0x08, 0xc8, 0xff, 0x4d, 0xd1, 0x83, 0xb2, 0x55, - 0x68, 0x74, 0x01, 0xdd, 0x5a, 0x05, 0xcf, 0x3c, 0xf8, 0x10, 0x0e, 0xb6, 0x4b, 0x1a, 0x9d, 0x43, - 0xd3, 0x64, 0xfb, 0x50, 0xfd, 0xeb, 0x9d, 0xde, 0x9a, 0xd4, 0x7e, 0x6f, 0x07, 0xb1, 0x33, 0x7b, - 0x76, 0xb2, 0xfa, 0x13, 0x3a, 0xab, 0x75, 0xe8, 0x3e, 0xac, 0x43, 0xf7, 0xf7, 0x3a, 0x74, 0xbf, - 0x6f, 0x42, 0xe7, 0x61, 0x13, 0x3a, 0x3f, 0x37, 0xa1, 0x73, 0xd3, 0xb6, 0x2f, 0x68, 0xda, 0x36, - 0x0f, 0xe0, 0xed, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0b, 0x22, 0x37, 0x8b, 0x57, 0x03, 0x00, - 0x00, + // 490 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0xcf, 0x6b, 0xdb, 0x30, + 0x14, 0xc7, 0xed, 0xe6, 0xa7, 0xe5, 0xa6, 0x5b, 0x45, 0x37, 0x9c, 0x1c, 0xdc, 0x60, 0x7a, 0x08, + 0x6c, 0xd8, 0x90, 0xc1, 0x18, 0xec, 0xb4, 0x96, 0xc2, 0x0a, 0x2b, 0x6c, 0x4e, 0x4e, 0xbd, 0x04, + 0xa5, 0x55, 0x53, 0x83, 0x2d, 0xa9, 0x92, 0x02, 0xc9, 0x7f, 0xb1, 0x7f, 0x65, 0xff, 0x45, 0x8e, + 0x3d, 0xee, 0x34, 0xb6, 0xe4, 0x1f, 0x19, 0x7a, 0x72, 0xba, 0x98, 0xf5, 0xd4, 0x4b, 0x22, 0xbd, + 0xcf, 0xf7, 0xfb, 0xac, 0xf7, 0xf4, 0x84, 0x5e, 0x65, 0xec, 0x96, 0x27, 0xe6, 0x47, 0x4c, 0x13, + 0x29, 0xae, 0x63, 0x21, 0xb9, 0xe6, 0xd8, 0xd7, 0x77, 0x84, 0x71, 0x15, 0x1b, 0xd0, 0xeb, 0x2a, + 0xcd, 0x25, 0x4d, 0x72, 0x32, 0xa5, 0xb9, 0x98, 0x26, 0x7a, 0x29, 0xa8, 0xb2, 0xba, 0xde, 0xd1, + 0x8c, 0xcf, 0x38, 0x2c, 0x13, 0xb3, 0xb2, 0xd1, 0xa8, 0x83, 0xfc, 0x0b, 0x76, 0xcb, 0x53, 0x7a, + 0x3f, 0xa7, 0x4a, 0x47, 0x3f, 0x6a, 0x68, 0xdf, 0xee, 0x95, 0xe0, 0x4c, 0x51, 0xfc, 0x1e, 0x21, + 0x48, 0x36, 0x51, 0x54, 0xab, 0xc0, 0xed, 0xd7, 0x06, 0xfe, 0xf0, 0x30, 0x2e, 0x3f, 0x79, 0xf5, + 0xc5, 0xa0, 0x11, 0xd5, 0xa7, 0xf5, 0xd5, 0xaf, 0x63, 0x27, 0xf5, 0xf2, 0x72, 0xaf, 0xf0, 0x09, + 0xea, 0x9c, 0xf1, 0x42, 0x70, 0x46, 0x99, 0x1e, 0x2f, 0x05, 0x0d, 0xf6, 0xfa, 0xee, 0xc0, 0x4b, + 0xab, 0x41, 0xfc, 0x16, 0x35, 0xe0, 0xc0, 0x41, 0xad, 0xef, 0x0e, 0xfc, 0xe1, 0xeb, 0x78, 0xa7, + 0x96, 0x78, 0x64, 0x08, 0x1c, 0xc6, 0x8a, 0x8c, 0x5a, 0xce, 0x73, 0xaa, 0x82, 0xfa, 0x13, 0xea, + 0xd4, 0x10, 0xab, 0x06, 0x11, 0xfe, 0x8c, 0x5e, 0x14, 0x54, 0xcb, 0xec, 0x7a, 0x52, 0x50, 0x4d, + 0x6e, 0x88, 0x26, 0x41, 0x03, 0x7c, 0xc7, 0x15, 0xdf, 0x25, 0x68, 0x2e, 0x4b, 0x09, 0x24, 0x38, + 0x28, 0x2a, 0x31, 0x3c, 0x44, 0x2d, 0x4d, 0xe4, 0xcc, 0x34, 0xa0, 0x09, 0x19, 0x82, 0x4a, 0x86, + 0xb1, 0x65, 0x60, 0xdd, 0x0a, 0xf1, 0x07, 0xe4, 0xd1, 0x05, 0x2d, 0x44, 0x4e, 0xa4, 0x0a, 0x5a, + 0xe0, 0xea, 0x55, 0x5c, 0xe7, 0x5b, 0x0a, 0xbe, 0x7f, 0x62, 0x9c, 0xa0, 0xc6, 0xfd, 0x9c, 0xca, + 0x65, 0xd0, 0x06, 0x57, 0xb7, 0xe2, 0xfa, 0x66, 0xc8, 0xa7, 0xaf, 0x17, 0xb6, 0x50, 0xd0, 0x45, + 0x0c, 0x79, 0x8f, 0xad, 0xc2, 0x5d, 0xd4, 0x2e, 0x32, 0x36, 0xd1, 0x59, 0x41, 0x03, 0xb7, 0xef, + 0x0e, 0x6a, 0x69, 0xab, 0xc8, 0xd8, 0x38, 0x2b, 0x28, 0x20, 0xb2, 0xb0, 0x68, 0xaf, 0x44, 0x64, + 0x01, 0xe8, 0x0d, 0x3a, 0x54, 0x73, 0x21, 0xb8, 0xd4, 0x6a, 0xa2, 0xee, 0x88, 0xbc, 0xc9, 0xd8, + 0x0c, 0xee, 0xa4, 0x9d, 0xbe, 0xdc, 0x82, 0x51, 0x19, 0x8f, 0x7c, 0xe4, 0x3d, 0x36, 0x3b, 0x3a, + 0x42, 0xf8, 0xff, 0x0e, 0x9a, 0xa9, 0xda, 0xe9, 0x4a, 0x74, 0x8e, 0x3a, 0x95, 0x72, 0x9f, 0x77, + 0xca, 0xe8, 0x00, 0xed, 0xef, 0xd6, 0x3f, 0x3c, 0x43, 0x75, 0xc8, 0xf6, 0xb1, 0xfc, 0xaf, 0x5e, + 0xcb, 0xce, 0x58, 0xf7, 0xba, 0x4f, 0x10, 0x3b, 0xe0, 0xa7, 0x27, 0xab, 0x3f, 0xa1, 0xb3, 0x5a, + 0x87, 0xee, 0xc3, 0x3a, 0x74, 0x7f, 0xaf, 0x43, 0xf7, 0xfb, 0x26, 0x74, 0x1e, 0x36, 0xa1, 0xf3, + 0x73, 0x13, 0x3a, 0x57, 0x4d, 0xfb, 0xdc, 0xa6, 0x4d, 0x78, 0x2d, 0xef, 0xfe, 0x06, 0x00, 0x00, + 0xff, 0xff, 0x5b, 0xc7, 0x91, 0xa9, 0x84, 0x03, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -629,6 +631,16 @@ func (m *StoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SupportsSharding { + i-- + if m.SupportsSharding { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if m.MaxTime != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MaxTime)) i-- @@ -842,6 +854,9 @@ func (m *StoreInfo) Size() (n int) { if m.MaxTime != 0 { n += 1 + sovRpc(uint64(m.MaxTime)) } + if m.SupportsSharding { + n += 2 + } return n } @@ -1351,6 +1366,26 @@ func (m *StoreInfo) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SupportsSharding", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.SupportsSharding = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) diff --git a/pkg/info/infopb/rpc.proto b/pkg/info/infopb/rpc.proto index 66a182a3e5..6f164932ab 100644 --- a/pkg/info/infopb/rpc.proto +++ b/pkg/info/infopb/rpc.proto @@ -55,6 +55,7 @@ message InfoResponse { message StoreInfo { int64 min_time = 1; int64 max_time = 2; + bool supports_sharding = 3; } // RulesInfo holds the metadata related to Rules API exposed by the component. diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go index c200eaaf78..82b7db880f 100644 --- a/pkg/query/endpointset.go +++ b/pkg/query/endpointset.go @@ -723,6 +723,17 @@ func (er *endpointRef) TimeRange() (mint, maxt int64) { return er.metadata.Store.MinTime, er.metadata.Store.MaxTime } +func (er *endpointRef) SupportsSharding() bool { + er.mtx.RLock() + defer er.mtx.RUnlock() + + if er.metadata == nil || er.metadata.Store == nil { + return false + } + + return er.metadata.Store.SupportsSharding +} + func (er *endpointRef) String() string { mint, maxt := er.TimeRange() return fmt.Sprintf("Addr: %s LabelSets: %v Mint: %d Maxt: %d", er.addr, labelpb.PromLabelSetsToString(er.LabelSets()), mint, maxt) diff --git a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go index 6f939166a4..b5fb097d31 100644 --- a/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go +++ b/pkg/query/internal/test-storeset-pre-v0.8.0/storeset.go @@ -204,6 +204,10 @@ func (s *storeRef) TimeRange() (int64, int64) { return s.minTime, s.maxTime } +func (s *storeRef) SupportsSharding() bool { + return false +} + func (s *storeRef) String() string { mint, maxt := s.TimeRange() return fmt.Sprintf("Addr: %s LabelSets: %v Mint: %d Maxt: %d", s.addr, labelpb.PromLabelSetsToString(s.LabelSets()), mint, maxt) diff --git a/pkg/query/querier.go b/pkg/query/querier.go index fcb939c8c7..361834c07d 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -34,7 +34,7 @@ import ( // replicaLabels at query time. // maxResolutionMillis controls downsampling resolution that is allowed (specified in milliseconds). // partialResponse controls `partialResponseDisabled` option of StoreAPI and partial response behavior of proxy. -type QueryableCreator func(deduplicate bool, replicaLabels []string, storeDebugMatchers [][]*labels.Matcher, maxResolutionMillis int64, partialResponse, enableQueryPushdown, skipChunks bool) storage.Queryable +type QueryableCreator func(deduplicate bool, replicaLabels []string, storeDebugMatchers [][]*labels.Matcher, maxResolutionMillis int64, partialResponse, enableQueryPushdown, skipChunks bool, shardInfo *storepb.ShardInfo) storage.Queryable // NewQueryableCreator creates QueryableCreator. func NewQueryableCreator(logger log.Logger, reg prometheus.Registerer, proxy storepb.StoreServer, maxConcurrentSelects int, selectTimeout time.Duration) QueryableCreator { @@ -42,7 +42,7 @@ func NewQueryableCreator(logger log.Logger, reg prometheus.Registerer, proxy sto extprom.WrapRegistererWithPrefix("concurrent_selects_", reg), ).NewHistogram(gate.DurationHistogramOpts) - return func(deduplicate bool, replicaLabels []string, storeDebugMatchers [][]*labels.Matcher, maxResolutionMillis int64, partialResponse, enableQueryPushdown, skipChunks bool) storage.Queryable { + return func(deduplicate bool, replicaLabels []string, storeDebugMatchers [][]*labels.Matcher, maxResolutionMillis int64, partialResponse, enableQueryPushdown, skipChunks bool, shardInfo *storepb.ShardInfo) storage.Queryable { return &queryable{ logger: logger, replicaLabels: replicaLabels, @@ -58,6 +58,7 @@ func NewQueryableCreator(logger log.Logger, reg prometheus.Registerer, proxy sto maxConcurrentSelects: maxConcurrentSelects, selectTimeout: selectTimeout, enableQueryPushdown: enableQueryPushdown, + shardInfo: shardInfo, } } } @@ -75,11 +76,12 @@ type queryable struct { maxConcurrentSelects int selectTimeout time.Duration enableQueryPushdown bool + shardInfo *storepb.ShardInfo } // Querier returns a new storage querier against the underlying proxy store API. func (q *queryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.storeDebugMatchers, q.proxy, q.deduplicate, q.maxResolutionMillis, q.partialResponse, q.enableQueryPushdown, q.skipChunks, q.gateProviderFn(), q.selectTimeout), nil + return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.storeDebugMatchers, q.proxy, q.deduplicate, q.maxResolutionMillis, q.partialResponse, q.enableQueryPushdown, q.skipChunks, q.gateProviderFn(), q.selectTimeout, q.shardInfo), nil } type querier struct { @@ -97,6 +99,7 @@ type querier struct { skipChunks bool selectGate gate.Gate selectTimeout time.Duration + shardInfo *storepb.ShardInfo } // newQuerier creates implementation of storage.Querier that fetches data from the proxy @@ -113,6 +116,7 @@ func newQuerier( partialResponse, enableQueryPushdown bool, skipChunks bool, selectGate gate.Gate, selectTimeout time.Duration, + shardInfo *storepb.ShardInfo, ) *querier { if logger == nil { logger = log.NewNopLogger() @@ -140,6 +144,7 @@ func newQuerier( partialResponse: partialResponse, skipChunks: skipChunks, enableQueryPushdown: enableQueryPushdown, + shardInfo: shardInfo, } } @@ -291,6 +296,7 @@ func (q *querier) selectFn(ctx context.Context, hints *storage.SelectHints, ms . if q.enableQueryPushdown { queryHints = storeHintsFromPromHints(hints) } + if err := q.proxy.Series(&storepb.SeriesRequest{ MinTime: hints.Start, MaxTime: hints.End, @@ -298,6 +304,7 @@ func (q *querier) selectFn(ctx context.Context, hints *storage.SelectHints, ms . MaxResolutionWindow: q.maxResolutionMillis, Aggregates: aggrs, QueryHints: queryHints, + ShardInfo: q.shardInfo, PartialResponseDisabled: !q.partialResponse, SkipChunks: q.skipChunks, Step: hints.Step, diff --git a/pkg/query/querier_test.go b/pkg/query/querier_test.go index d726d120a6..2b127e95a0 100644 --- a/pkg/query/querier_test.go +++ b/pkg/query/querier_test.go @@ -45,7 +45,7 @@ func TestQueryableCreator_MaxResolution(t *testing.T) { queryableCreator := NewQueryableCreator(nil, nil, testProxy, 2, 5*time.Second) oneHourMillis := int64(1*time.Hour) / int64(time.Millisecond) - queryable := queryableCreator(false, nil, nil, oneHourMillis, false, false, false) + queryable := queryableCreator(false, nil, nil, oneHourMillis, false, false, false, nil) q, err := queryable.Querier(context.Background(), 0, 42) testutil.Ok(t, err) @@ -72,7 +72,7 @@ func TestQuerier_DownsampledData(t *testing.T) { } timeout := 10 * time.Second - q := NewQueryableCreator(nil, nil, testProxy, 2, timeout)(false, nil, nil, 9999999, false, false, false) + q := NewQueryableCreator(nil, nil, testProxy, 2, timeout)(false, nil, nil, 9999999, false, false, false, nil) engine := promql.NewEngine( promql.EngineOpts{ MaxSamples: math.MaxInt32, @@ -364,7 +364,7 @@ func TestQuerier_Select_AfterPromQL(t *testing.T) { g := gate.New(2) mq := &mockedQueryable{ Creator: func(mint, maxt int64) storage.Querier { - return newQuerier(context.Background(), nil, mint, maxt, tcase.replicaLabels, nil, tcase.storeAPI, sc.dedup, 0, true, false, false, g, timeout) + return newQuerier(context.Background(), nil, mint, maxt, tcase.replicaLabels, nil, tcase.storeAPI, sc.dedup, 0, true, false, false, g, timeout, nil) }, } t.Cleanup(func() { @@ -608,7 +608,7 @@ func TestQuerier_Select(t *testing.T) { {dedup: true, expected: []series{tcase.expectedAfterDedup}}, } { g := gate.New(2) - q := newQuerier(context.Background(), nil, tcase.mint, tcase.maxt, tcase.replicaLabels, nil, tcase.storeAPI, sc.dedup, 0, true, false, false, g, timeout) + q := newQuerier(context.Background(), nil, tcase.mint, tcase.maxt, tcase.replicaLabels, nil, tcase.storeAPI, sc.dedup, 0, true, false, false, g, timeout, nil) t.Cleanup(func() { testutil.Ok(t, q.Close()) }) t.Run(fmt.Sprintf("dedup=%v", sc.dedup), func(t *testing.T) { @@ -837,7 +837,7 @@ func TestQuerierWithDedupUnderstoodByPromQL_Rate(t *testing.T) { timeout := 100 * time.Second g := gate.New(2) - q := newQuerier(context.Background(), logger, realSeriesWithStaleMarkerMint, realSeriesWithStaleMarkerMaxt, []string{"replica"}, nil, s, false, 0, true, false, false, g, timeout) + q := newQuerier(context.Background(), logger, realSeriesWithStaleMarkerMint, realSeriesWithStaleMarkerMaxt, []string{"replica"}, nil, s, false, 0, true, false, false, g, timeout, nil) t.Cleanup(func() { testutil.Ok(t, q.Close()) }) @@ -907,7 +907,7 @@ func TestQuerierWithDedupUnderstoodByPromQL_Rate(t *testing.T) { timeout := 5 * time.Second g := gate.New(2) - q := newQuerier(context.Background(), logger, realSeriesWithStaleMarkerMint, realSeriesWithStaleMarkerMaxt, []string{"replica"}, nil, s, true, 0, true, false, false, g, timeout) + q := newQuerier(context.Background(), logger, realSeriesWithStaleMarkerMint, realSeriesWithStaleMarkerMaxt, []string{"replica"}, nil, s, true, 0, true, false, false, g, timeout, nil) t.Cleanup(func() { testutil.Ok(t, q.Close()) }) diff --git a/pkg/query/query_test.go b/pkg/query/query_test.go index 3b1d19d004..2921622c3c 100644 --- a/pkg/query/query_test.go +++ b/pkg/query/query_test.go @@ -54,7 +54,7 @@ func TestQuerier_Proxy(t *testing.T) { name: fmt.Sprintf("store number %v", i), }) } - return q(true, nil, nil, 0, false, false, false) + return q(true, nil, nil, 0, false, false, false, nil) } for _, fn := range files { diff --git a/pkg/query/test.go b/pkg/query/test.go index 782aa4dccb..d43d2883b7 100644 --- a/pkg/query/test.go +++ b/pkg/query/test.go @@ -653,5 +653,9 @@ func (i inProcessClient) TimeRange() (mint, maxt int64) { return r.MinTime, r.MaxTime } +func (i inProcessClient) SupportsSharding() bool { + return false +} + func (i inProcessClient) String() string { return i.name } func (i inProcessClient) Addr() string { return i.name } diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index 4778f51218..d4a9847f47 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -204,6 +204,7 @@ type Config struct { RequestLoggingDecision string DownstreamURL string ForwardHeaders []string + NumShards int } // QueryRangeConfig holds the config for query range tripperware. diff --git a/pkg/queryfrontend/queryrange_codec.go b/pkg/queryfrontend/queryrange_codec.go index 975b6b4567..5ee3a5b373 100644 --- a/pkg/queryfrontend/queryrange_codec.go +++ b/pkg/queryfrontend/queryrange_codec.go @@ -6,6 +6,7 @@ package queryfrontend import ( "bytes" "context" + "encoding/json" "math" "net/http" "net/url" @@ -117,6 +118,11 @@ func (c queryRangeCodec) DecodeRequest(_ context.Context, r *http.Request, forwa return nil, err } + result.ShardInfo, err = parseShardInfo(r.Form, queryv1.ShardInfoParam) + if err != nil { + return nil, err + } + result.Query = r.FormValue("query") result.Path = r.URL.Path @@ -165,6 +171,14 @@ func (c queryRangeCodec) EncodeRequest(ctx context.Context, r queryrange.Request params[queryv1.StoreMatcherParam] = matchersToStringSlice(thanosReq.StoreMatchers) } + if thanosReq.ShardInfo != nil { + data, err := encodeShardInfo(thanosReq.ShardInfo) + if err != nil { + return nil, err + } + params[queryv1.ShardInfoParam] = []string{data} + } + req, err := http.NewRequest(http.MethodPost, thanosReq.Path, bytes.NewBufferString(params.Encode())) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, "error creating request: %s", err.Error()) @@ -246,6 +260,24 @@ func parseMatchersParam(ss url.Values, matcherParam string) ([][]*labels.Matcher return matchers, nil } +func parseShardInfo(ss url.Values, key string) (*storepb.ShardInfo, error) { + data, ok := ss[key] + if !ok { + return nil, nil + } + + if len(data) == 0 { + return nil, nil + } + + var info storepb.ShardInfo + if err := json.Unmarshal([]byte(data[0]), &info); err != nil { + return nil, err + } + + return &info, nil +} + func encodeTime(t int64) string { f := float64(t) / 1.0e3 return strconv.FormatFloat(f, 'f', -1, 64) @@ -263,3 +295,16 @@ func matchersToStringSlice(storeMatchers [][]*labels.Matcher) []string { } return res } + +func encodeShardInfo(info *storepb.ShardInfo) (string, error) { + if info == nil { + return "", nil + } + + data, err := json.Marshal(info) + if err != nil { + return "", err + } + + return string(data), nil +} diff --git a/pkg/queryfrontend/request.go b/pkg/queryfrontend/request.go index 8e3703d1a8..8ff4fdc250 100644 --- a/pkg/queryfrontend/request.go +++ b/pkg/queryfrontend/request.go @@ -6,6 +6,8 @@ package queryfrontend import ( "time" + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" "github.com/prometheus/prometheus/model/labels" @@ -46,6 +48,7 @@ type ThanosQueryRangeRequest struct { CachingOptions queryrange.CachingOptions Headers []*RequestHeader Stats string + ShardInfo *storepb.ShardInfo } // IsDedupEnabled returns true if deduplication is enabled. @@ -93,6 +96,13 @@ func (r *ThanosQueryRangeRequest) WithQuery(query string) queryrange.Request { return &q } +// WithShardInfo clones the current request with a different shard info. +func (r *ThanosQueryRangeRequest) WithShardInfo(info *storepb.ShardInfo) queryrange.Request { + q := *r + q.ShardInfo = info + return &q +} + // LogToSpan writes information about this request to an OpenTracing span. func (r *ThanosQueryRangeRequest) LogToSpan(sp opentracing.Span) { fields := []otlog.Field{ diff --git a/pkg/queryfrontend/roundtrip.go b/pkg/queryfrontend/roundtrip.go index a47a5df0c1..73fecf232a 100644 --- a/pkg/queryfrontend/roundtrip.go +++ b/pkg/queryfrontend/roundtrip.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/thanos-io/thanos/pkg/querysharding" + "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -52,7 +54,11 @@ func NewTripperware(config Config, reg prometheus.Registerer, logger log.Logger) queryRangeCodec := NewThanosQueryRangeCodec(config.QueryRangeConfig.PartialResponseStrategy) labelsCodec := NewThanosLabelsCodec(config.LabelsConfig.PartialResponseStrategy, config.DefaultTimeRange) - queryRangeTripperware, err := newQueryRangeTripperware(config.QueryRangeConfig, queryRangeLimits, queryRangeCodec, + queryRangeTripperware, err := newQueryRangeTripperware( + config.QueryRangeConfig, + queryRangeLimits, + queryRangeCodec, + config.NumShards, prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "query_range"}, reg), logger, config.ForwardHeaders) if err != nil { return nil, err @@ -137,6 +143,7 @@ func newQueryRangeTripperware( config QueryRangeConfig, limits queryrange.Limits, codec *queryRangeCodec, + numShards int, reg prometheus.Registerer, logger log.Logger, forwardHeaders []string, @@ -204,6 +211,13 @@ func newQueryRangeTripperware( ) } + if numShards > 0 { + queryRangeMiddleware = append( + queryRangeMiddleware, + PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec), + ) + } + return func(next http.RoundTripper) http.RoundTripper { rt := queryrange.NewRoundTripper(next, codec, forwardHeaders, queryRangeMiddleware...) return queryrange.RoundTripFunc(func(r *http.Request) (*http.Response, error) { diff --git a/pkg/queryfrontend/shard_query.go b/pkg/queryfrontend/shard_query.go new file mode 100644 index 0000000000..e2bab7a94b --- /dev/null +++ b/pkg/queryfrontend/shard_query.go @@ -0,0 +1,85 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// This is a modified copy from +// https://github.com/cortexproject/cortex/blob/master/pkg/querier/queryrange/split_by_interval.go. + +package queryfrontend + +import ( + "context" + + "github.com/cortexproject/cortex/pkg/querier/queryrange" + "github.com/thanos-io/thanos/pkg/querysharding" + "github.com/thanos-io/thanos/pkg/store/storepb" +) + +// PromQLShardingMiddleware creates a new Middleware that shards PromQL aggregations using grouping labels. +func PromQLShardingMiddleware(queryAnalyzer *querysharding.QueryAnalyzer, numShards int, limits queryrange.Limits, merger queryrange.Merger) queryrange.Middleware { + return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler { + return querySharder{ + next: next, + limits: limits, + queryAnalyzer: queryAnalyzer, + numShards: numShards, + merger: merger, + } + }) +} + +type querySharder struct { + next queryrange.Handler + limits queryrange.Limits + + queryAnalyzer *querysharding.QueryAnalyzer + numShards int + merger queryrange.Merger +} + +func (s querySharder) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) { + analysis, err := s.queryAnalyzer.Analyze(r.GetQuery()) + if err != nil { + return nil, err + } + + if !analysis.IsShardable() { + return s.next.Do(ctx, r) + } + + reqs := s.shardQuery(r, analysis) + + reqResps, err := queryrange.DoRequests(ctx, s.next, reqs, s.limits) + if err != nil { + return nil, err + } + + resps := make([]queryrange.Response, 0, len(reqResps)) + for _, reqResp := range reqResps { + resps = append(resps, reqResp.Response) + } + + response, err := s.merger.MergeResponse(resps...) + if err != nil { + return nil, err + } + return response, nil +} + +func (s querySharder) shardQuery(r queryrange.Request, analysis querysharding.QueryAnalysis) []queryrange.Request { + tr, ok := r.(*ThanosQueryRangeRequest) + if !ok { + return []queryrange.Request{tr} + } + + reqs := make([]queryrange.Request, s.numShards) + for i := 0; i < s.numShards; i++ { + reqs[i] = tr.WithShardInfo(&storepb.ShardInfo{ + TotalShards: int64(s.numShards), + ShardIndex: int64(i), + By: analysis.ShardBy(), + Labels: analysis.ShardingLabels(), + }) + } + + return reqs +} diff --git a/pkg/querysharding/analysis.go b/pkg/querysharding/analysis.go new file mode 100644 index 0000000000..90f9fcc806 --- /dev/null +++ b/pkg/querysharding/analysis.go @@ -0,0 +1,140 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package querysharding + +var excludedLabels = []string{"le"} + +type QueryAnalysis struct { + // Labels to shard on + shardingLabels []string + + // When set to true, sharding is `by` shardingLabels, + // otherwise it is `without` shardingLabels. + shardBy bool +} + +func nonShardableQuery() QueryAnalysis { + return QueryAnalysis{ + shardingLabels: nil, + } +} + +func newShardableByLabels(labels []string, by bool) QueryAnalysis { + labels = without(labels, excludedLabels) + + return QueryAnalysis{ + shardBy: by, + shardingLabels: labels, + } +} + +func (q QueryAnalysis) scopeToLabels(labels []string, by bool) QueryAnalysis { + labels = without(labels, excludedLabels) + + if q.shardingLabels == nil { + return QueryAnalysis{ + shardBy: by, + shardingLabels: labels, + } + } + + if by { + return QueryAnalysis{ + shardBy: true, + shardingLabels: intersect(q.shardingLabels, labels), + } + } + + return QueryAnalysis{ + shardBy: false, + shardingLabels: union(q.shardingLabels, labels), + } +} + +func (q QueryAnalysis) IsShardable() bool { + return len(q.shardingLabels) > 0 +} + +func (q QueryAnalysis) ShardingLabels() []string { + if len(q.shardingLabels) == 0 { + return nil + } + + return q.shardingLabels +} + +func (q QueryAnalysis) ShardBy() bool { + return q.shardBy +} + +func intersect(sliceA, sliceB []string) []string { + if len(sliceA) == 0 || len(sliceB) == 0 { + return []string{} + } + + mapA := make(map[string]struct{}, len(sliceA)) + for _, s := range sliceA { + mapA[s] = struct{}{} + } + + mapB := make(map[string]struct{}, len(sliceB)) + for _, s := range sliceB { + mapB[s] = struct{}{} + } + + result := make([]string, 0) + for k := range mapA { + if _, ok := mapB[k]; ok { + result = append(result, k) + } + } + + return result +} + +func without(sliceA, sliceB []string) []string { + if sliceA == nil { + return nil + } + + if len(sliceA) == 0 || len(sliceB) == 0 { + return []string{} + } + + keyMap := make(map[string]struct{}, len(sliceA)) + for _, s := range sliceA { + keyMap[s] = struct{}{} + } + for _, s := range sliceB { + delete(keyMap, s) + } + + result := make([]string, 0, len(keyMap)) + for k := range keyMap { + result = append(result, k) + } + + return result +} + +func union(sliceA, sliceB []string) []string { + if len(sliceA) == 0 || len(sliceB) == 0 { + return []string{} + } + + keyMap := make(map[string]struct{}, len(sliceA)) + for _, s := range sliceA { + keyMap[s] = struct{}{} + } + for _, s := range sliceB { + keyMap[s] = struct{}{} + } + + result := make([]string, 0, len(keyMap)) + for k := range keyMap { + result = append(result, k) + } + + return result +} diff --git a/pkg/querysharding/analyzer.go b/pkg/querysharding/analyzer.go new file mode 100644 index 0000000000..e3fe99f951 --- /dev/null +++ b/pkg/querysharding/analyzer.go @@ -0,0 +1,109 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package querysharding + +import ( + "fmt" + + "github.com/prometheus/prometheus/promql/parser" +) + +// QueryAnalyzer is an analyzer which determines +// whether a PromQL Query is shardable and using which labels. +type QueryAnalyzer struct{} + +var nonShardableFuncs = []string{ + "label_join", + "label_replace", +} + +// NewQueryAnalyzer creates a new QueryAnalyzer. +func NewQueryAnalyzer() *QueryAnalyzer { + return &QueryAnalyzer{} +} + +// Analyze analyzes a query and returns a QueryAnalysis. + +// Analyze uses the following algorithm: +// * if a query has subqueries, such as label_join or label_replace, +// then treat the query as non shardable. +// * if the query's root expression has grouping labels, +// then treat the query as shardable by those labels. +// * if the query's root expression has no grouping labels, +// then walk the query and find the least common labelset +// used in grouping expressions. If non-empty, treat the query +// as shardable by those labels. +// * otherwise, treat the query as non-shardable. +func (a *QueryAnalyzer) Analyze(query string) (QueryAnalysis, error) { + expr, err := parser.ParseExpr(query) + if err != nil { + return nonShardableQuery(), err + } + + isShardable := true + var analysis QueryAnalysis + parser.Inspect(expr, func(node parser.Node, nodes []parser.Node) error { + switch n := node.(type) { + case *parser.SubqueryExpr: + isShardable = false + return fmt.Errorf("expressions with subqueries are not shardable") + case *parser.Call: + if n.Func != nil && contains(n.Func.Name, nonShardableFuncs) { + isShardable = false + return fmt.Errorf("expressions with %s are not shardable", n.Func.Name) + } + case *parser.BinaryExpr: + analysis = analysis.scopeToLabels(n.VectorMatching.MatchingLabels, n.VectorMatching.On) + case *parser.AggregateExpr: + labels := make([]string, 0) + if len(n.Grouping) > 0 { + labels = n.Grouping + } + + analysis = analysis.scopeToLabels(labels, !n.Without) + } + + return nil + }) + + if !isShardable { + return nonShardableQuery(), nil + } + + rootAnalysis := analyzeRootExpression(expr) + if rootAnalysis.IsShardable() && rootAnalysis.shardBy { + return rootAnalysis, nil + } + + return analysis, nil +} + +func analyzeRootExpression(node parser.Node) QueryAnalysis { + switch n := node.(type) { + case *parser.BinaryExpr: + if n.VectorMatching != nil && n.VectorMatching.On { + return newShardableByLabels(n.VectorMatching.MatchingLabels, n.VectorMatching.On) + } else { + return nonShardableQuery() + } + case *parser.AggregateExpr: + if len(n.Grouping) == 0 { + return nonShardableQuery() + } + + return newShardableByLabels(n.Grouping, !n.Without) + } + + return nonShardableQuery() +} + +func contains(needle string, haystack []string) bool { + for _, item := range haystack { + if needle == item { + return true + } + } + + return false +} diff --git a/pkg/querysharding/analyzer_test.go b/pkg/querysharding/analyzer_test.go new file mode 100644 index 0000000000..9dbb9c948d --- /dev/null +++ b/pkg/querysharding/analyzer_test.go @@ -0,0 +1,193 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package querysharding + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAnalyzeQuery(t *testing.T) { + type testCase struct { + name string + expression string + shardingLabels []string + } + + nonShardable := []testCase{ + { + name: "aggregation", + expression: "sum(http_requests_total)", + }, + { + name: "outer aggregation with no grouping", + expression: "count(sum by (pod) (http_requests_total))", + }, + { + name: "outer aggregation with without grouping", + expression: "count(sum without (pod) (http_requests_total))", + }, + { + name: "aggregate expression with subquery", + expression: `sum by (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`, + }, + { + name: "aggregate without expression with subquery", + expression: `sum without (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`, + }, + { + name: "binary expression", + expression: `http_requests_total{code="400"} / http_requests_total`, + }, + { + name: "binary expression with empty vector matching", + expression: `http_requests_total{code="400"} / on () http_requests_total`, + }, + { + name: "binary aggregation with different grouping labels", + expression: `sum by (pod) (http_requests_total{code="400"}) / sum by (cluster) (http_requests_total)`, + }, + { + name: "binary expression with vector matching and subquery", + expression: `http_requests_total{code="400"} / on (pod) label_replace(metric, "dst_label", "$1", "src_label", "re")`, + }, + { + name: "multiple binary expressions", + expression: `(http_requests_total{code="400"} + http_requests_total{code="500"}) / http_requests_total`, + }, + { + name: "multiple binary expressions with empty vector matchers", + expression: ` +(http_requests_total{code="400"} + on (cluster, pod) http_requests_total{code="500"}) +/ on () +http_requests_total`, + }, + } + + shardableByLabels := []testCase{ + { + name: "aggregation with grouping", + expression: "sum by (pod) (http_requests_total)", + shardingLabels: []string{"pod"}, + }, + { + name: "multiple aggregations with grouping", + expression: "max by (pod) (sum by (pod, cluster) (http_requests_total))", + shardingLabels: []string{"pod"}, + }, + { + name: "binary expression with vector matching", + expression: `http_requests_total{code="400"} / on (pod) http_requests_total`, + shardingLabels: []string{"pod"}, + }, + { + name: "binary aggregation with same grouping labels", + expression: `sum by (pod) (http_requests_total{code="400"}) / sum by (pod) (http_requests_total)`, + shardingLabels: []string{"pod"}, + }, + { + name: "binary expression with vector matching and grouping", + expression: `sum by (cluster, pod) (http_requests_total{code="400"}) / on (pod) sum by (cluster, pod) (http_requests_total)`, + shardingLabels: []string{"pod"}, + }, + { + name: "multiple binary expressions with vector matchers", + expression: ` +(http_requests_total{code="400"} + on (cluster, pod) http_requests_total{code="500"}) +/ on (pod) +http_requests_total`, + shardingLabels: []string{"pod"}, + }, + { + name: "multiple binary expressions with grouping", + expression: ` +sum by (container) ( + (http_requests_total{code="400"} + on (cluster, pod, container) http_requests_total{code="500"}) + / on (pod, container) + http_requests_total +)`, + shardingLabels: []string{"container"}, + }, + { + name: "multiple binary expressions with grouping", + expression: `(http_requests_total{code="400"} + on (pod) http_requests_total{code="500"}) / on (cluster, pod) http_requests_total`, + shardingLabels: []string{"cluster", "pod"}, + }, + { + name: "histogram quantile", + expression: "histogram_quantile(0.95, sum(rate(metric[1m])) by (le, cluster))", + shardingLabels: []string{"cluster"}, + }, + } + + shardableWithoutLabels := []testCase{ + { + name: "aggregation without grouping", + expression: "sum without (pod) (http_requests_total)", + shardingLabels: []string{"pod"}, + }, + { + name: "multiple aggregations with without grouping", + expression: "max without (pod) (sum without (pod, cluster) (http_requests_total))", + shardingLabels: []string{"pod", "cluster"}, + }, + { + name: "binary expression with without vector matching and grouping", + expression: `sum without (cluster, pod) (http_requests_total{code="400"}) / ignoring (pod) sum without (cluster, pod) (http_requests_total)`, + shardingLabels: []string{"pod", "cluster"}, + }, + { + name: "multiple binary expressions with without grouping", + expression: `(http_requests_total{code="400"} + ignoring (pod) http_requests_total{code="500"}) / ignoring (cluster, pod) http_requests_total`, + shardingLabels: []string{"cluster", "pod"}, + }, + { + name: "multiple binary expressions with without vector matchers", + expression: ` +(http_requests_total{code="400"} + ignoring (cluster, pod) http_requests_total{code="500"}) +/ ignoring (pod) +http_requests_total`, + shardingLabels: []string{"cluster", "pod"}, + }, + } + + for _, test := range nonShardable { + t.Run(test.name, func(t *testing.T) { + analyzer := NewQueryAnalyzer() + analysis, err := analyzer.Analyze(test.expression) + require.NoError(t, err) + require.False(t, analysis.IsShardable()) + }) + } + + for _, test := range shardableByLabels { + t.Run(test.name, func(t *testing.T) { + analyzer := NewQueryAnalyzer() + analysis, err := analyzer.Analyze(test.expression) + require.NoError(t, err) + require.True(t, analysis.IsShardable()) + require.True(t, analysis.ShardBy()) + + sort.Strings(test.shardingLabels) + sort.Strings(analysis.ShardingLabels()) + require.Equal(t, test.shardingLabels, analysis.ShardingLabels()) + }) + } + + for _, test := range shardableWithoutLabels { + t.Run(test.name, func(t *testing.T) { + analyzer := NewQueryAnalyzer() + analysis, err := analyzer.Analyze(test.expression) + require.NoError(t, err) + require.True(t, analysis.IsShardable()) + require.False(t, analysis.ShardBy()) + + sort.Strings(test.shardingLabels) + sort.Strings(analysis.ShardingLabels()) + require.Equal(t, test.shardingLabels, analysis.ShardingLabels()) + }) + } +} diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 61acbb2fe1..64f3163341 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -784,6 +784,7 @@ func blockSeries( skipChunks bool, // If true, chunks are not loaded. minTime, maxTime int64, // Series must have data in this time range to be returned. loadAggregates []storepb.Aggr, // List of aggregates to load when loading chunks. + shardMatcher *storepb.ShardMatcher, ) (storepb.SeriesSet, *queryStats, error) { ps, err := indexr.ExpandedPostings(ctx, matchers) if err != nil { @@ -814,6 +815,7 @@ func blockSeries( lset labels.Labels chks []chunks.Meta ) + for _, id := range ps { ok, err := indexr.LoadSeriesForTime(id, &symbolizedLset, &chks, skipChunks, minTime, maxTime) if err != nil { @@ -824,7 +826,17 @@ func blockSeries( continue } + if err := indexr.LookupLabelsSymbols(symbolizedLset, &lset); err != nil { + return nil, nil, errors.Wrap(err, "Lookup labels symbols") + } + + if !shardMatcher.MatchesLabels(lset) { + continue + } + s := seriesEntry{} + s.lset = labelpb.ExtendSortedLabels(lset, extLset) + if !skipChunks { // Schedule loading chunks. s.refs = make([]chunks.ChunkRef, 0, len(chks)) @@ -847,11 +859,7 @@ func blockSeries( return nil, nil, errors.Wrap(err, "exceeded chunks limit") } } - if err := indexr.LookupLabelsSymbols(symbolizedLset, &lset); err != nil { - return nil, nil, errors.Wrap(err, "Lookup labels symbols") - } - s.lset = labelpb.ExtendSortedLabels(lset, extLset) res = append(res, s) } @@ -1070,6 +1078,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie req.SkipChunks, req.MinTime, req.MaxTime, req.Aggregates, + req.ShardInfo.Matcher(), ) if err != nil { return errors.Wrapf(err, "fetch series for block %s", b.meta.ULID) @@ -1272,7 +1281,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq result = strutil.MergeSlices(res, extRes) } else { - seriesSet, _, err := blockSeries(newCtx, b.extLset, indexr, nil, reqSeriesMatchers, nil, seriesLimiter, true, req.Start, req.End, nil) + seriesSet, _, err := blockSeries(newCtx, b.extLset, indexr, nil, reqSeriesMatchers, nil, seriesLimiter, true, req.Start, req.End, nil, nil) if err != nil { return errors.Wrapf(err, "fetch series for block %s", b.meta.ULID) } @@ -1403,7 +1412,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR } result = res } else { - seriesSet, _, err := blockSeries(newCtx, b.extLset, indexr, nil, reqSeriesMatchers, nil, seriesLimiter, true, req.Start, req.End, nil) + seriesSet, _, err := blockSeries(newCtx, b.extLset, indexr, nil, reqSeriesMatchers, nil, seriesLimiter, true, req.Start, req.End, nil, nil) if err != nil { return errors.Wrapf(err, "fetch series for block %s", b.meta.ULID) } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index d0557718d4..d5a2062573 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -2291,7 +2291,7 @@ func benchmarkBlockSeriesWithConcurrency(b *testing.B, concurrency int, blockMet indexReader := blk.indexReader() chunkReader := blk.chunkReader() - seriesSet, _, err := blockSeries(context.Background(), nil, indexReader, chunkReader, matchers, chunksLimiter, seriesLimiter, req.SkipChunks, req.MinTime, req.MaxTime, req.Aggregates) + seriesSet, _, err := blockSeries(context.Background(), nil, indexReader, chunkReader, matchers, chunksLimiter, seriesLimiter, req.SkipChunks, req.MinTime, req.MaxTime, req.Aggregates, nil) testutil.Ok(b, err) // Ensure at least 1 series has been returned (as expected). diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 7a8a46c85e..93991e06a2 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -195,11 +195,13 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie return nil } - if r.QueryHints != nil && r.QueryHints.IsSafeToExecute() { + shardMatcher := r.ShardInfo.Matcher() + if r.QueryHints != nil && r.QueryHints.IsSafeToExecute() && !shardMatcher.IsSharded() { return p.queryPrometheus(s, r) } q := &prompb.Query{StartTimestampMs: r.MinTime, EndTimestampMs: r.MaxTime} + level.Info(p.logger).Log("min_time", r.MinTime, "max_time", r.MaxTime) for _, m := range matchers { pm := &prompb.LabelMatcher{Name: m.Name, Value: m.Value} @@ -237,7 +239,7 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie if !strings.HasPrefix(contentType, "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse") { return errors.Errorf("not supported remote read content type: %s", contentType) } - return p.handleStreamedPrometheusResponse(s, httpResp, queryPrometheusSpan, extLset) + return p.handleStreamedPrometheusResponse(s, shardMatcher, httpResp, queryPrometheusSpan, extLset) } func (p *PrometheusStore) queryPrometheus(s storepb.Store_SeriesServer, r *storepb.SeriesRequest) error { @@ -351,7 +353,13 @@ func (p *PrometheusStore) handleSampledPrometheusResponse(s storepb.Store_Series return nil } -func (p *PrometheusStore) handleStreamedPrometheusResponse(s storepb.Store_SeriesServer, httpResp *http.Response, querySpan tracing.Span, extLset labels.Labels) error { +func (p *PrometheusStore) handleStreamedPrometheusResponse( + s storepb.Store_SeriesServer, + shardMatcher *storepb.ShardMatcher, + httpResp *http.Response, + querySpan tracing.Span, + extLset labels.Labels, +) error { level.Debug(p.logger).Log("msg", "started handling ReadRequest_STREAMED_XOR_CHUNKS streamed read response.") framesNum := 0 @@ -387,6 +395,10 @@ func (p *PrometheusStore) handleStreamedPrometheusResponse(s storepb.Store_Serie framesNum++ for _, series := range res.ChunkedSeries { + if !shardMatcher.MatchesZLabels(series.Labels) { + continue + } + seriesStats.CountSeries(series.Labels) thanosChks := make([]storepb.AggrChunk, len(series.Chunks)) for i, chk := range series.Chunks { @@ -408,12 +420,13 @@ func (p *PrometheusStore) handleStreamedPrometheusResponse(s storepb.Store_Serie series.Chunks[i].Data = nil } - if err := s.Send(storepb.NewSeriesResponse(&storepb.Series{ + r := storepb.NewSeriesResponse(&storepb.Series{ Labels: labelpb.ZLabelsFromPromLabels( labelpb.ExtendSortedLabels(labelpb.ZLabelsToPromLabels(series.Labels), extLset), ), Chunks: thanosChks, - })); err != nil { + }) + if err := s.Send(r); err != nil { return err } } @@ -424,6 +437,7 @@ func (p *PrometheusStore) handleStreamedPrometheusResponse(s storepb.Store_Serie querySpan.SetTag("processed.samples", seriesStats.Samples) querySpan.SetTag("processed.bytes", bodySizer.BytesCount()) level.Debug(p.logger).Log("msg", "handled ReadRequest_STREAMED_XOR_CHUNKS request.", "frames", framesNum) + return nil } diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 330cab85b6..97a92f531f 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -48,6 +48,9 @@ type Client interface { // TimeRange returns minimum and maximum time range of data in the store. TimeRange() (mint int64, maxt int64) + // SupportsSharding returns true if sharding is supported by the underlying store. + SupportsSharding() bool + String() string // Addr returns address of a Client. Addr() string @@ -276,6 +279,7 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe MaxResolutionWindow: r.MaxResolutionWindow, SkipChunks: r.SkipChunks, QueryHints: r.QueryHints, + ShardInfo: r.ShardInfo, PartialResponseDisabled: r.PartialResponseDisabled, } wg = &sync.WaitGroup{} @@ -326,8 +330,22 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe // Schedule streamSeriesSet that translates gRPC streamed response // into seriesSet (if series) or respCh if warnings. - seriesSet = append(seriesSet, startStreamSeriesSet(seriesCtx, reqLogger, span, closeSeries, - wg, sc, respSender, st.String(), !r.PartialResponseDisabled, s.responseTimeout, s.metrics.emptyStreamResponses)) + seriesSet = append(seriesSet, + startStreamSeriesSet( + seriesCtx, + reqLogger, + span, + closeSeries, + wg, + sc, + respSender, + st.String(), + !r.PartialResponseDisabled, + s.responseTimeout, + s.metrics.emptyStreamResponses, + st.SupportsSharding(), + r.ShardInfo, + )) } level.Debug(reqLogger).Log("msg", "Series: started fanout streams", "status", strings.Join(storeDebugMsgs, ";")) @@ -422,6 +440,8 @@ func startStreamSeriesSet( partialResponse bool, responseTimeout time.Duration, emptyStreamResponses prometheus.Counter, + storeSupportsSharding bool, + shardInfo *storepb.ShardInfo, ) *streamSeriesSet { s := &streamSeriesSet{ ctx: ctx, @@ -470,6 +490,14 @@ func startStreamSeriesSet( } } }() + + shardMatcher := shardInfo.Matcher() + applySharding := shardInfo != nil && !storeSupportsSharding + if applySharding { + msg := "Applying series sharding in the proxy since there is not support in the underlying store" + level.Debug(logger).Log("msg", msg, "store", name) + } + // The `defer` only executed when function return, we do `defer cancel` in for loop, // so make the loop body as a function, release timers created by context as early. handleRecvResponse := func() (next bool) { @@ -503,6 +531,10 @@ func startStreamSeriesSet( } if series := rr.r.GetSeries(); series != nil { + if applySharding && !shardMatcher.MatchesZLabels(series.Labels) { + return true + } + seriesStats.Count(series) select { diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index 303a87e027..125c8d4197 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -37,9 +37,10 @@ type testClient struct { // Just to pass interface check. storepb.StoreClient - labelSets []labels.Labels - minTime int64 - maxTime int64 + labelSets []labels.Labels + minTime int64 + maxTime int64 + supportsSharding bool } func (c testClient) LabelSets() []labels.Labels { @@ -50,6 +51,10 @@ func (c testClient) TimeRange() (int64, int64) { return c.minTime, c.maxTime } +func (c testClient) SupportsSharding() bool { + return c.supportsSharding +} + func (c testClient) String() string { return "test" } @@ -468,6 +473,44 @@ func TestProxyStore_Series(t *testing.T) { storeDebugMatchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "__address__", "foo")}}, expectedWarningsLen: 1, // No stores match. }, + { + title: "sharded series response", + storeAPIs: []Client{ + &testClient{ + StoreClient: &mockedStoreAPI{ + RespSeries: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "a"), []sample{{0, 0}, {2, 1}, {3, 2}}), + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{0, 0}, {2, 1}, {3, 2}}), + storeSeriesResponse(t, labels.FromStrings("a", "c"), []sample{{0, 0}, {2, 1}, {3, 2}}), + }, + }, + minTime: 1, + maxTime: 300, + labelSets: []labels.Labels{labels.FromStrings("ext", "1")}, + }, + }, + req: &storepb.SeriesRequest{ + MinTime: 1, + MaxTime: 300, + Matchers: []storepb.LabelMatcher{{Name: "ext", Value: "1", Type: storepb.LabelMatcher_EQ}}, + ShardInfo: &storepb.ShardInfo{ + ShardIndex: 0, + TotalShards: 2, + By: true, + Labels: []string{"a"}, + }, + }, + expectedSeries: []rawSeries{ + { + lset: labels.FromStrings("a", "a"), + chunks: [][]sample{{{0, 0}, {2, 1}, {3, 2}}}, + }, + { + lset: labels.FromStrings("a", "b"), + chunks: [][]sample{{{0, 0}, {2, 1}, {3, 2}}}, + }, + }, + }, } { if ok := t.Run(tc.title, func(t *testing.T) { diff --git a/pkg/store/storepb/rpc.pb.go b/pkg/store/storepb/rpc.pb.go index bc2e98b272..bf670d0f63 100644 --- a/pkg/store/storepb/rpc.pb.go +++ b/pkg/store/storepb/rpc.pb.go @@ -289,6 +289,9 @@ type SeriesRequest struct { // query_hints are the hints coming from the PromQL engine when // requesting a storage.SeriesSet for a given expression. QueryHints *QueryHints `protobuf:"bytes,12,opt,name=query_hints,json=queryHints,proto3" json:"query_hints,omitempty"` + // shard_info is used by the querier to request a specific + // shard of blocks instead of entire blocks. + ShardInfo *ShardInfo `protobuf:"bytes,13,opt,name=shard_info,json=shardInfo,proto3" json:"shard_info,omitempty"` } func (m *SeriesRequest) Reset() { *m = SeriesRequest{} } @@ -369,6 +372,51 @@ func (m *QueryHints) XXX_DiscardUnknown() { var xxx_messageInfo_QueryHints proto.InternalMessageInfo +// ShardInfo are the parameters used to shard series in Stores. +type ShardInfo struct { + // The index of the current shard. + ShardIndex int64 `protobuf:"varint,1,opt,name=shard_index,json=shardIndex,proto3" json:"shard_index,omitempty"` + // The total number of shards. + TotalShards int64 `protobuf:"varint,2,opt,name=total_shards,json=totalShards,proto3" json:"total_shards,omitempty"` + // Group by or without labels. + By bool `protobuf:"varint,3,opt,name=by,proto3" json:"by,omitempty"` + // Labels on which to partition series. + Labels []string `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty"` +} + +func (m *ShardInfo) Reset() { *m = ShardInfo{} } +func (m *ShardInfo) String() string { return proto.CompactTextString(m) } +func (*ShardInfo) ProtoMessage() {} +func (*ShardInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_a938d55a388af629, []int{6} +} +func (m *ShardInfo) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardInfo.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardInfo.Merge(m, src) +} +func (m *ShardInfo) XXX_Size() int { + return m.Size() +} +func (m *ShardInfo) XXX_DiscardUnknown() { + xxx_messageInfo_ShardInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardInfo proto.InternalMessageInfo + type Func struct { // The function or aggregation name Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -378,7 +426,7 @@ func (m *Func) Reset() { *m = Func{} } func (m *Func) String() string { return proto.CompactTextString(m) } func (*Func) ProtoMessage() {} func (*Func) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{6} + return fileDescriptor_a938d55a388af629, []int{7} } func (m *Func) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -418,7 +466,7 @@ func (m *Grouping) Reset() { *m = Grouping{} } func (m *Grouping) String() string { return proto.CompactTextString(m) } func (*Grouping) ProtoMessage() {} func (*Grouping) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{7} + return fileDescriptor_a938d55a388af629, []int{8} } func (m *Grouping) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -455,7 +503,7 @@ func (m *Range) Reset() { *m = Range{} } func (m *Range) String() string { return proto.CompactTextString(m) } func (*Range) ProtoMessage() {} func (*Range) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{8} + return fileDescriptor_a938d55a388af629, []int{9} } func (m *Range) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -496,7 +544,7 @@ func (m *SeriesResponse) Reset() { *m = SeriesResponse{} } func (m *SeriesResponse) String() string { return proto.CompactTextString(m) } func (*SeriesResponse) ProtoMessage() {} func (*SeriesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{9} + return fileDescriptor_a938d55a388af629, []int{10} } func (m *SeriesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -599,7 +647,7 @@ func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } func (m *LabelNamesRequest) String() string { return proto.CompactTextString(m) } func (*LabelNamesRequest) ProtoMessage() {} func (*LabelNamesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{10} + return fileDescriptor_a938d55a388af629, []int{11} } func (m *LabelNamesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -641,7 +689,7 @@ func (m *LabelNamesResponse) Reset() { *m = LabelNamesResponse{} } func (m *LabelNamesResponse) String() string { return proto.CompactTextString(m) } func (*LabelNamesResponse) ProtoMessage() {} func (*LabelNamesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{11} + return fileDescriptor_a938d55a388af629, []int{12} } func (m *LabelNamesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -688,7 +736,7 @@ func (m *LabelValuesRequest) Reset() { *m = LabelValuesRequest{} } func (m *LabelValuesRequest) String() string { return proto.CompactTextString(m) } func (*LabelValuesRequest) ProtoMessage() {} func (*LabelValuesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{12} + return fileDescriptor_a938d55a388af629, []int{13} } func (m *LabelValuesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -730,7 +778,7 @@ func (m *LabelValuesResponse) Reset() { *m = LabelValuesResponse{} } func (m *LabelValuesResponse) String() string { return proto.CompactTextString(m) } func (*LabelValuesResponse) ProtoMessage() {} func (*LabelValuesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_a938d55a388af629, []int{13} + return fileDescriptor_a938d55a388af629, []int{14} } func (m *LabelValuesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -768,6 +816,7 @@ func init() { proto.RegisterType((*InfoResponse)(nil), "thanos.InfoResponse") proto.RegisterType((*SeriesRequest)(nil), "thanos.SeriesRequest") proto.RegisterType((*QueryHints)(nil), "thanos.QueryHints") + proto.RegisterType((*ShardInfo)(nil), "thanos.ShardInfo") proto.RegisterType((*Func)(nil), "thanos.Func") proto.RegisterType((*Grouping)(nil), "thanos.Grouping") proto.RegisterType((*Range)(nil), "thanos.Range") @@ -781,84 +830,89 @@ func init() { func init() { proto.RegisterFile("store/storepb/rpc.proto", fileDescriptor_a938d55a388af629) } var fileDescriptor_a938d55a388af629 = []byte{ - // 1230 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5f, 0x6f, 0x13, 0x47, - 0x10, 0xf7, 0xdd, 0xf9, 0xfc, 0x67, 0x9c, 0xa4, 0xc7, 0x62, 0xe0, 0x62, 0x24, 0xc7, 0xba, 0xaa, - 0x52, 0x84, 0xa8, 0xdd, 0x9a, 0x0a, 0xa9, 0x15, 0x2f, 0x49, 0x30, 0x24, 0x2a, 0x31, 0x65, 0x9d, - 0x90, 0x96, 0xaa, 0xb2, 0xce, 0xce, 0x72, 0x3e, 0xe1, 0xfb, 0xc3, 0xed, 0x5e, 0xc1, 0xaf, 0xed, - 0x7b, 0x55, 0xf5, 0x23, 0x54, 0xfd, 0x10, 0xfd, 0x08, 0xbc, 0x95, 0xc7, 0xaa, 0x0f, 0xa8, 0x85, - 0x2f, 0x52, 0xed, 0x9f, 0xb3, 0x7d, 0x69, 0x80, 0xa2, 0xf0, 0x72, 0xda, 0x99, 0xdf, 0xec, 0xec, - 0xcc, 0xfc, 0x76, 0xe6, 0x16, 0x2e, 0x51, 0x16, 0x25, 0xa4, 0x23, 0xbe, 0xf1, 0xa8, 0x93, 0xc4, - 0xe3, 0x76, 0x9c, 0x44, 0x2c, 0x42, 0x25, 0x36, 0x71, 0xc3, 0x88, 0x36, 0xd6, 0xf3, 0x06, 0x6c, - 0x16, 0x13, 0x2a, 0x4d, 0x1a, 0x75, 0x2f, 0xf2, 0x22, 0xb1, 0xec, 0xf0, 0x95, 0xd2, 0xb6, 0xf2, - 0x1b, 0xe2, 0x24, 0x0a, 0x4e, 0xec, 0x53, 0x2e, 0xa7, 0xee, 0x88, 0x4c, 0x4f, 0x42, 0x5e, 0x14, - 0x79, 0x53, 0xd2, 0x11, 0xd2, 0x28, 0x7d, 0xd8, 0x71, 0xc3, 0x99, 0x84, 0x9c, 0x0f, 0x60, 0xf5, - 0x28, 0xf1, 0x19, 0xc1, 0x84, 0xc6, 0x51, 0x48, 0x89, 0xf3, 0xa3, 0x06, 0x2b, 0x4a, 0xf3, 0x38, - 0x25, 0x94, 0xa1, 0x2d, 0x00, 0xe6, 0x07, 0x84, 0x92, 0xc4, 0x27, 0xd4, 0xd6, 0x5a, 0xc6, 0x66, - 0xad, 0x7b, 0x99, 0xef, 0x0e, 0x08, 0x9b, 0x90, 0x94, 0x0e, 0xc7, 0x51, 0x3c, 0x6b, 0x1f, 0xf8, - 0x01, 0x19, 0x08, 0x93, 0xed, 0xe2, 0xb3, 0x17, 0x1b, 0x05, 0xbc, 0xb4, 0x09, 0x5d, 0x84, 0x12, - 0x23, 0xa1, 0x1b, 0x32, 0x5b, 0x6f, 0x69, 0x9b, 0x55, 0xac, 0x24, 0x64, 0x43, 0x39, 0x21, 0xf1, - 0xd4, 0x1f, 0xbb, 0xb6, 0xd1, 0xd2, 0x36, 0x0d, 0x9c, 0x89, 0xce, 0x2a, 0xd4, 0xf6, 0xc2, 0x87, - 0x91, 0x8a, 0xc1, 0xf9, 0x45, 0x87, 0x15, 0x29, 0xcb, 0x28, 0xd1, 0x18, 0x4a, 0x22, 0xd1, 0x2c, - 0xa0, 0xd5, 0xb6, 0x2c, 0x6c, 0xfb, 0x0e, 0xd7, 0x6e, 0xdf, 0xe0, 0x21, 0xfc, 0xf5, 0x62, 0xe3, - 0x33, 0xcf, 0x67, 0x93, 0x74, 0xd4, 0x1e, 0x47, 0x41, 0x47, 0x1a, 0x7c, 0xec, 0x47, 0x6a, 0xd5, - 0x89, 0x1f, 0x79, 0x9d, 0x5c, 0xcd, 0xda, 0x0f, 0xc4, 0x6e, 0xac, 0x5c, 0xa3, 0x75, 0xa8, 0x04, - 0x7e, 0x38, 0xe4, 0x89, 0x88, 0xc0, 0x0d, 0x5c, 0x0e, 0xfc, 0x90, 0x67, 0x2a, 0x20, 0xf7, 0xa9, - 0x84, 0x54, 0xe8, 0x81, 0xfb, 0x54, 0x40, 0x1d, 0xa8, 0x0a, 0xaf, 0x07, 0xb3, 0x98, 0xd8, 0xc5, - 0x96, 0xb6, 0xb9, 0xd6, 0x3d, 0x97, 0x45, 0x37, 0xc8, 0x00, 0xbc, 0xb0, 0x41, 0xd7, 0x01, 0xc4, - 0x81, 0x43, 0x4a, 0x18, 0xb5, 0x4d, 0x91, 0xcf, 0x7c, 0x87, 0x0c, 0x69, 0x40, 0x98, 0x2a, 0x6b, - 0x75, 0xaa, 0x64, 0xea, 0xfc, 0x56, 0x84, 0x55, 0x59, 0xf2, 0x8c, 0xaa, 0xe5, 0x80, 0xb5, 0xd7, - 0x07, 0xac, 0xe7, 0x03, 0xbe, 0xce, 0x21, 0x36, 0x9e, 0x90, 0x84, 0xda, 0x86, 0x38, 0xbd, 0x9e, - 0xab, 0xe6, 0xbe, 0x04, 0x55, 0x00, 0x73, 0x5b, 0xd4, 0x85, 0x0b, 0xdc, 0x65, 0x42, 0x68, 0x34, - 0x4d, 0x99, 0x1f, 0x85, 0xc3, 0x27, 0x7e, 0x78, 0x1c, 0x3d, 0x11, 0x49, 0x1b, 0xf8, 0x7c, 0xe0, - 0x3e, 0xc5, 0x73, 0xec, 0x48, 0x40, 0xe8, 0x2a, 0x80, 0xeb, 0x79, 0x09, 0xf1, 0x5c, 0x46, 0x64, - 0xae, 0x6b, 0xdd, 0x95, 0xec, 0xb4, 0x2d, 0xcf, 0x4b, 0xf0, 0x12, 0x8e, 0xbe, 0x80, 0xf5, 0xd8, - 0x4d, 0x98, 0xef, 0x4e, 0xf9, 0x29, 0x82, 0xf9, 0xe1, 0xb1, 0x4f, 0xdd, 0xd1, 0x94, 0x1c, 0xdb, - 0xa5, 0x96, 0xb6, 0x59, 0xc1, 0x97, 0x94, 0x41, 0x76, 0x33, 0x6e, 0x2a, 0x18, 0x7d, 0x7b, 0xca, - 0x5e, 0xca, 0x12, 0x97, 0x11, 0x6f, 0x66, 0x97, 0x05, 0x2d, 0x1b, 0xd9, 0xc1, 0x5f, 0xe5, 0x7d, - 0x0c, 0x94, 0xd9, 0x7f, 0x9c, 0x67, 0x00, 0xda, 0x80, 0x1a, 0x7d, 0xe4, 0xc7, 0xc3, 0xf1, 0x24, - 0x0d, 0x1f, 0x51, 0xbb, 0x22, 0x42, 0x01, 0xae, 0xda, 0x11, 0x1a, 0x74, 0x05, 0xcc, 0x89, 0x1f, - 0x32, 0x6a, 0x57, 0x5b, 0x9a, 0x28, 0xa8, 0xec, 0xc0, 0x76, 0xd6, 0x81, 0xed, 0xad, 0x70, 0x86, - 0xa5, 0x09, 0x42, 0x50, 0xa4, 0x8c, 0xc4, 0x36, 0x88, 0xb2, 0x89, 0x35, 0xaa, 0x83, 0x99, 0xb8, - 0xa1, 0x47, 0xec, 0x9a, 0x50, 0x4a, 0x01, 0x5d, 0x83, 0xda, 0xe3, 0x94, 0x24, 0xb3, 0xa1, 0xf4, - 0xbd, 0x22, 0x7c, 0xa3, 0x2c, 0x8b, 0x7b, 0x1c, 0xda, 0xe5, 0x08, 0x86, 0xc7, 0xf3, 0xb5, 0xf3, - 0xab, 0x06, 0xb0, 0x80, 0x44, 0xe8, 0x8c, 0xc4, 0xc3, 0xc0, 0x9f, 0x4e, 0x7d, 0xaa, 0xae, 0x09, - 0x70, 0xd5, 0xbe, 0xd0, 0xa0, 0x16, 0x14, 0x1f, 0xa6, 0xe1, 0x58, 0xdc, 0x92, 0xda, 0x82, 0x9c, - 0x5b, 0x69, 0x38, 0xc6, 0x02, 0x41, 0x57, 0xa1, 0xe2, 0x25, 0x51, 0x1a, 0xfb, 0xa1, 0x27, 0xb8, - 0xae, 0x75, 0xad, 0xcc, 0xea, 0xb6, 0xd2, 0xe3, 0xb9, 0x05, 0xfa, 0x30, 0x4b, 0xc5, 0x14, 0xa6, - 0xf3, 0x4e, 0xc5, 0x5c, 0xa9, 0x32, 0x73, 0x1a, 0x50, 0xe4, 0x07, 0xf0, 0x5a, 0x84, 0xae, 0xba, - 0xbd, 0x55, 0x2c, 0xd6, 0x4e, 0x17, 0x2a, 0x99, 0x5b, 0xb4, 0x06, 0xfa, 0x68, 0x26, 0xd0, 0x0a, - 0xd6, 0x47, 0x33, 0x3e, 0x59, 0xd4, 0x1c, 0xe0, 0x37, 0xb7, 0x9a, 0xb5, 0xae, 0xb3, 0x01, 0xa6, - 0xf0, 0xcf, 0x0d, 0x72, 0x99, 0x2a, 0xc9, 0xf9, 0x49, 0x83, 0xb5, 0xac, 0x79, 0xd4, 0x4c, 0xd9, - 0x84, 0xd2, 0x7c, 0xc8, 0xf1, 0x48, 0xd7, 0xe6, 0x5d, 0x2b, 0xb4, 0xbb, 0x05, 0xac, 0x70, 0xd4, - 0x80, 0xf2, 0x13, 0x37, 0x09, 0x79, 0xfe, 0x62, 0xa0, 0xed, 0x16, 0x70, 0xa6, 0x40, 0x57, 0x33, - 0xe6, 0x8d, 0xd7, 0x33, 0xbf, 0x5b, 0x50, 0xdc, 0x6f, 0x57, 0xa0, 0x94, 0x10, 0x9a, 0x4e, 0x99, - 0xf3, 0xbb, 0x0e, 0xe7, 0x44, 0xbb, 0xf5, 0xdd, 0x60, 0xd1, 0xd1, 0x6f, 0xec, 0x00, 0xed, 0x0c, - 0x1d, 0xa0, 0x9f, 0xb1, 0x03, 0xea, 0x60, 0x52, 0xe6, 0x26, 0x4c, 0x4d, 0x3f, 0x29, 0x20, 0x0b, - 0x0c, 0x12, 0x1e, 0xab, 0x01, 0xc0, 0x97, 0x8b, 0x46, 0x30, 0xdf, 0xde, 0x08, 0xcb, 0x83, 0xa8, - 0xf4, 0xff, 0x07, 0x91, 0x93, 0x00, 0x5a, 0xae, 0x9c, 0xa2, 0xb3, 0x0e, 0x26, 0xbf, 0x3e, 0xf2, - 0x0f, 0x51, 0xc5, 0x52, 0x40, 0x0d, 0xa8, 0x28, 0xa6, 0xa8, 0xad, 0x0b, 0x60, 0x2e, 0x2f, 0x62, - 0x35, 0xde, 0x1a, 0xab, 0xf3, 0x87, 0xae, 0x0e, 0xbd, 0xef, 0x4e, 0xd3, 0x05, 0x5f, 0x75, 0x30, - 0xc5, 0x0d, 0x54, 0x17, 0x58, 0x0a, 0x6f, 0x66, 0x51, 0x3f, 0x03, 0x8b, 0xc6, 0xfb, 0x62, 0xb1, - 0x78, 0x0a, 0x8b, 0xe6, 0x29, 0x2c, 0x96, 0xde, 0x8d, 0xc5, 0xf2, 0x3b, 0xb0, 0x98, 0xc2, 0xf9, - 0x5c, 0x41, 0x15, 0x8d, 0x17, 0xa1, 0xf4, 0xbd, 0xd0, 0x28, 0x1e, 0x95, 0xf4, 0xbe, 0x88, 0xbc, - 0xf2, 0x1d, 0x54, 0xe7, 0x7f, 0x65, 0x54, 0x83, 0xf2, 0x61, 0xff, 0xcb, 0xfe, 0xdd, 0xa3, 0xbe, - 0x55, 0x40, 0x55, 0x30, 0xef, 0x1d, 0xf6, 0xf0, 0x37, 0x96, 0x86, 0x2a, 0x50, 0xc4, 0x87, 0x77, - 0x7a, 0x96, 0xce, 0x2d, 0x06, 0x7b, 0x37, 0x7b, 0x3b, 0x5b, 0xd8, 0x32, 0xb8, 0xc5, 0xe0, 0xe0, - 0x2e, 0xee, 0x59, 0x45, 0xae, 0xc7, 0xbd, 0x9d, 0xde, 0xde, 0xfd, 0x9e, 0x65, 0x72, 0xfd, 0xcd, - 0xde, 0xf6, 0xe1, 0x6d, 0xab, 0x74, 0x65, 0x1b, 0x8a, 0xfc, 0xb7, 0x86, 0xca, 0x60, 0xe0, 0xad, - 0x23, 0xe9, 0x75, 0xe7, 0xee, 0x61, 0xff, 0xc0, 0xd2, 0xb8, 0x6e, 0x70, 0xb8, 0x6f, 0xe9, 0x7c, - 0xb1, 0xbf, 0xd7, 0xb7, 0x0c, 0xb1, 0xd8, 0xfa, 0x5a, 0xba, 0x13, 0x56, 0x3d, 0x6c, 0x99, 0xdd, - 0x1f, 0x74, 0x30, 0x45, 0x8c, 0xe8, 0x53, 0x28, 0xf2, 0x67, 0x10, 0x3a, 0x9f, 0x55, 0x74, 0xe9, - 0x91, 0xd4, 0xa8, 0xe7, 0x95, 0xaa, 0x7e, 0x9f, 0x43, 0x49, 0xce, 0x2f, 0x74, 0x21, 0x3f, 0xcf, - 0xb2, 0x6d, 0x17, 0x4f, 0xaa, 0xe5, 0xc6, 0x4f, 0x34, 0xb4, 0x03, 0xb0, 0xe8, 0x2b, 0xb4, 0x9e, - 0x63, 0x71, 0x79, 0x4a, 0x35, 0x1a, 0xa7, 0x41, 0xea, 0xfc, 0x5b, 0x50, 0x5b, 0xa2, 0x15, 0xe5, - 0x4d, 0x73, 0xcd, 0xd3, 0xb8, 0x7c, 0x2a, 0x26, 0xfd, 0x74, 0xfb, 0xb0, 0x26, 0x9e, 0xa5, 0xbc, - 0x2b, 0x64, 0x31, 0x6e, 0x40, 0x0d, 0x93, 0x20, 0x62, 0x44, 0xe8, 0xd1, 0x3c, 0xfd, 0xe5, 0xd7, - 0x6b, 0xe3, 0xc2, 0x09, 0xad, 0x7a, 0xe5, 0x16, 0xb6, 0x3f, 0x7a, 0xf6, 0x4f, 0xb3, 0xf0, 0xec, - 0x65, 0x53, 0x7b, 0xfe, 0xb2, 0xa9, 0xfd, 0xfd, 0xb2, 0xa9, 0xfd, 0xfc, 0xaa, 0x59, 0x78, 0xfe, - 0xaa, 0x59, 0xf8, 0xf3, 0x55, 0xb3, 0xf0, 0xa0, 0xac, 0x1e, 0xda, 0xa3, 0x92, 0xb8, 0x33, 0xd7, - 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x5a, 0xb7, 0x5e, 0x64, 0xd2, 0x0b, 0x00, 0x00, + // 1298 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x13, 0x47, + 0x17, 0xf6, 0x7a, 0xbd, 0xfe, 0x38, 0x4e, 0xf2, 0x9a, 0xc1, 0xc0, 0xc6, 0x48, 0x8e, 0xdf, 0x7d, + 0xf5, 0x4a, 0x11, 0xa2, 0x36, 0x35, 0x15, 0x52, 0x2b, 0x6e, 0x92, 0x60, 0x48, 0x54, 0x62, 0xca, + 0x38, 0x21, 0x2d, 0x55, 0x65, 0xad, 0xed, 0xc9, 0x7a, 0xc5, 0x7a, 0x77, 0xd9, 0x99, 0x6d, 0xe2, + 0xdb, 0xf6, 0xbe, 0xaa, 0xfa, 0x13, 0xfa, 0x2b, 0xfa, 0x13, 0xb8, 0x2b, 0x57, 0x55, 0xd5, 0x0b, + 0xd4, 0xc2, 0x1f, 0xa9, 0xe6, 0x63, 0xd7, 0xde, 0x34, 0x40, 0x11, 0xdc, 0x44, 0x73, 0x9e, 0xe7, + 0xcc, 0x99, 0xf3, 0xed, 0x0d, 0x5c, 0xa1, 0x2c, 0x88, 0x48, 0x47, 0xfc, 0x0d, 0x47, 0x9d, 0x28, + 0x1c, 0xb7, 0xc3, 0x28, 0x60, 0x01, 0x2a, 0xb2, 0xa9, 0xed, 0x07, 0xb4, 0xb1, 0x9e, 0x55, 0x60, + 0xf3, 0x90, 0x50, 0xa9, 0xd2, 0xa8, 0x3b, 0x81, 0x13, 0x88, 0x63, 0x87, 0x9f, 0x14, 0xda, 0xca, + 0x5e, 0x08, 0xa3, 0x60, 0x76, 0xe6, 0x9e, 0x32, 0xe9, 0xd9, 0x23, 0xe2, 0x9d, 0xa5, 0x9c, 0x20, + 0x70, 0x3c, 0xd2, 0x11, 0xd2, 0x28, 0x3e, 0xee, 0xd8, 0xfe, 0x5c, 0x52, 0xd6, 0x7f, 0x60, 0xf5, + 0x28, 0x72, 0x19, 0xc1, 0x84, 0x86, 0x81, 0x4f, 0x89, 0xf5, 0xbd, 0x06, 0x2b, 0x0a, 0x79, 0x1a, + 0x13, 0xca, 0xd0, 0x16, 0x00, 0x73, 0x67, 0x84, 0x92, 0xc8, 0x25, 0xd4, 0xd4, 0x5a, 0xfa, 0x66, + 0xb5, 0x7b, 0x95, 0xdf, 0x9e, 0x11, 0x36, 0x25, 0x31, 0x1d, 0x8e, 0x83, 0x70, 0xde, 0x3e, 0x70, + 0x67, 0x64, 0x20, 0x54, 0xb6, 0x0b, 0xcf, 0x5e, 0x6c, 0xe4, 0xf0, 0xd2, 0x25, 0x74, 0x19, 0x8a, + 0x8c, 0xf8, 0xb6, 0xcf, 0xcc, 0x7c, 0x4b, 0xdb, 0xac, 0x60, 0x25, 0x21, 0x13, 0x4a, 0x11, 0x09, + 0x3d, 0x77, 0x6c, 0x9b, 0x7a, 0x4b, 0xdb, 0xd4, 0x71, 0x22, 0x5a, 0xab, 0x50, 0xdd, 0xf3, 0x8f, + 0x03, 0xe5, 0x83, 0xf5, 0x53, 0x1e, 0x56, 0xa4, 0x2c, 0xbd, 0x44, 0x63, 0x28, 0x8a, 0x40, 0x13, + 0x87, 0x56, 0xdb, 0x32, 0xb1, 0xed, 0xfb, 0x1c, 0xdd, 0xbe, 0xcd, 0x5d, 0xf8, 0xe3, 0xc5, 0xc6, + 0x27, 0x8e, 0xcb, 0xa6, 0xf1, 0xa8, 0x3d, 0x0e, 0x66, 0x1d, 0xa9, 0xf0, 0x91, 0x1b, 0xa8, 0x53, + 0x27, 0x7c, 0xe2, 0x74, 0x32, 0x39, 0x6b, 0x3f, 0x16, 0xb7, 0xb1, 0x32, 0x8d, 0xd6, 0xa1, 0x3c, + 0x73, 0xfd, 0x21, 0x0f, 0x44, 0x38, 0xae, 0xe3, 0xd2, 0xcc, 0xf5, 0x79, 0xa4, 0x82, 0xb2, 0x4f, + 0x25, 0xa5, 0x5c, 0x9f, 0xd9, 0xa7, 0x82, 0xea, 0x40, 0x45, 0x58, 0x3d, 0x98, 0x87, 0xc4, 0x2c, + 0xb4, 0xb4, 0xcd, 0xb5, 0xee, 0x85, 0xc4, 0xbb, 0x41, 0x42, 0xe0, 0x85, 0x0e, 0xba, 0x05, 0x20, + 0x1e, 0x1c, 0x52, 0xc2, 0xa8, 0x69, 0x88, 0x78, 0xd2, 0x1b, 0xd2, 0xa5, 0x01, 0x61, 0x2a, 0xad, + 0x15, 0x4f, 0xc9, 0xd4, 0xfa, 0xad, 0x00, 0xab, 0x32, 0xe5, 0x49, 0xa9, 0x96, 0x1d, 0xd6, 0x5e, + 0xef, 0x70, 0x3e, 0xeb, 0xf0, 0x2d, 0x4e, 0xb1, 0xf1, 0x94, 0x44, 0xd4, 0xd4, 0xc5, 0xeb, 0xf5, + 0x4c, 0x36, 0xf7, 0x25, 0xa9, 0x1c, 0x48, 0x75, 0x51, 0x17, 0x2e, 0x71, 0x93, 0x11, 0xa1, 0x81, + 0x17, 0x33, 0x37, 0xf0, 0x87, 0x27, 0xae, 0x3f, 0x09, 0x4e, 0x44, 0xd0, 0x3a, 0xbe, 0x38, 0xb3, + 0x4f, 0x71, 0xca, 0x1d, 0x09, 0x0a, 0x5d, 0x07, 0xb0, 0x1d, 0x27, 0x22, 0x8e, 0xcd, 0x88, 0x8c, + 0x75, 0xad, 0xbb, 0x92, 0xbc, 0xb6, 0xe5, 0x38, 0x11, 0x5e, 0xe2, 0xd1, 0x67, 0xb0, 0x1e, 0xda, + 0x11, 0x73, 0x6d, 0x8f, 0xbf, 0x22, 0x2a, 0x3f, 0x9c, 0xb8, 0xd4, 0x1e, 0x79, 0x64, 0x62, 0x16, + 0x5b, 0xda, 0x66, 0x19, 0x5f, 0x51, 0x0a, 0x49, 0x67, 0xdc, 0x51, 0x34, 0xfa, 0xfa, 0x9c, 0xbb, + 0x94, 0x45, 0x36, 0x23, 0xce, 0xdc, 0x2c, 0x89, 0xb2, 0x6c, 0x24, 0x0f, 0x7f, 0x91, 0xb5, 0x31, + 0x50, 0x6a, 0xff, 0x30, 0x9e, 0x10, 0x68, 0x03, 0xaa, 0xf4, 0x89, 0x1b, 0x0e, 0xc7, 0xd3, 0xd8, + 0x7f, 0x42, 0xcd, 0xb2, 0x70, 0x05, 0x38, 0xb4, 0x23, 0x10, 0x74, 0x0d, 0x8c, 0xa9, 0xeb, 0x33, + 0x6a, 0x56, 0x5a, 0x9a, 0x48, 0xa8, 0x9c, 0xc0, 0x76, 0x32, 0x81, 0xed, 0x2d, 0x7f, 0x8e, 0xa5, + 0x0a, 0x42, 0x50, 0xa0, 0x8c, 0x84, 0x26, 0x88, 0xb4, 0x89, 0x33, 0xaa, 0x83, 0x11, 0xd9, 0xbe, + 0x43, 0xcc, 0xaa, 0x00, 0xa5, 0x80, 0x6e, 0x42, 0xf5, 0x69, 0x4c, 0xa2, 0xf9, 0x50, 0xda, 0x5e, + 0x11, 0xb6, 0x51, 0x12, 0xc5, 0x43, 0x4e, 0xed, 0x72, 0x06, 0xc3, 0xd3, 0xf4, 0x8c, 0x6e, 0x00, + 0xd0, 0xa9, 0x1d, 0x4d, 0x86, 0xae, 0x7f, 0x1c, 0x98, 0xab, 0xe2, 0xce, 0xa2, 0x21, 0x39, 0x23, + 0x26, 0xab, 0x42, 0x93, 0xa3, 0xf5, 0xb3, 0x06, 0xb0, 0x30, 0x26, 0x82, 0x65, 0x24, 0x1c, 0xce, + 0x5c, 0xcf, 0x73, 0xa9, 0x6a, 0x2c, 0xe0, 0xd0, 0xbe, 0x40, 0x50, 0x0b, 0x0a, 0xc7, 0xb1, 0x3f, + 0x16, 0x7d, 0x55, 0x5d, 0x94, 0xf3, 0x6e, 0xec, 0x8f, 0xb1, 0x60, 0xd0, 0x75, 0x28, 0x3b, 0x51, + 0x10, 0x87, 0xae, 0xef, 0x88, 0xee, 0xa8, 0x76, 0x6b, 0x89, 0xd6, 0x3d, 0x85, 0xe3, 0x54, 0x03, + 0xfd, 0x2f, 0x09, 0xde, 0x10, 0xaa, 0xe9, 0x6c, 0x63, 0x0e, 0xaa, 0x5c, 0x58, 0x27, 0x50, 0x49, + 0x9d, 0x17, 0x2e, 0xaa, 0x18, 0x27, 0xe4, 0x34, 0x75, 0x51, 0xf2, 0x13, 0x72, 0x8a, 0xfe, 0x0b, + 0x2b, 0x2c, 0x60, 0xb6, 0x37, 0x14, 0x18, 0x55, 0x23, 0x50, 0x15, 0x98, 0x30, 0x43, 0xd1, 0x1a, + 0xe4, 0x47, 0x73, 0x31, 0xcc, 0x65, 0x9c, 0x1f, 0xcd, 0xf9, 0xd2, 0x52, 0x2b, 0xa6, 0xd0, 0xd2, + 0xf9, 0xd2, 0x92, 0x92, 0xd5, 0x80, 0x02, 0x8f, 0x8c, 0x97, 0xcd, 0xb7, 0xd5, 0xa0, 0x55, 0xb0, + 0x38, 0x5b, 0x5d, 0x28, 0x27, 0xf1, 0x28, 0x7b, 0xda, 0x39, 0xf6, 0xf4, 0x8c, 0xbd, 0x0d, 0x30, + 0x44, 0x60, 0x5c, 0x21, 0x93, 0x62, 0x25, 0x59, 0x3f, 0x68, 0xb0, 0x96, 0xcc, 0xb9, 0x5a, 0x7f, + 0x9b, 0x50, 0x4c, 0xf7, 0x31, 0x4f, 0xd1, 0x5a, 0x5a, 0x4f, 0x81, 0xee, 0xe6, 0xb0, 0xe2, 0x51, + 0x03, 0x4a, 0x27, 0x76, 0xe4, 0xf3, 0xc4, 0x8b, 0xdd, 0xbb, 0x9b, 0xc3, 0x09, 0x80, 0xae, 0x27, + 0x4d, 0xaa, 0xbf, 0xbe, 0x49, 0x77, 0x73, 0xaa, 0x4d, 0xb7, 0xcb, 0x50, 0x8c, 0x08, 0x8d, 0x3d, + 0x66, 0xfd, 0x92, 0x87, 0x0b, 0x62, 0x33, 0xf4, 0xed, 0xd9, 0x62, 0xf9, 0xbc, 0x71, 0x58, 0xb5, + 0xf7, 0x18, 0xd6, 0xfc, 0x7b, 0x0e, 0x6b, 0x1d, 0x0c, 0xca, 0xec, 0x88, 0xa9, 0x45, 0x2d, 0x05, + 0x54, 0x03, 0x9d, 0xf8, 0x13, 0xb5, 0xab, 0xf8, 0x71, 0x31, 0xb3, 0xc6, 0xdb, 0x67, 0x76, 0x79, + 0x67, 0x16, 0xff, 0xfd, 0xce, 0xb4, 0x22, 0x40, 0xcb, 0x99, 0x53, 0xe5, 0xac, 0x83, 0xc1, 0xdb, + 0x47, 0xfe, 0x98, 0x55, 0xb0, 0x14, 0x50, 0x03, 0xca, 0xaa, 0x52, 0xbc, 0x5f, 0x39, 0x91, 0xca, + 0x0b, 0x5f, 0xf5, 0xb7, 0xfa, 0x6a, 0xfd, 0x9a, 0x57, 0x8f, 0x3e, 0xb2, 0xbd, 0x78, 0x51, 0xaf, + 0x3a, 0x18, 0xa2, 0x03, 0x55, 0x03, 0x4b, 0xe1, 0xcd, 0x55, 0xcc, 0xbf, 0x47, 0x15, 0xf5, 0x0f, + 0x55, 0xc5, 0xc2, 0x39, 0x55, 0x34, 0xce, 0xa9, 0x62, 0xf1, 0xdd, 0xaa, 0x58, 0x7a, 0x87, 0x2a, + 0xc6, 0x70, 0x31, 0x93, 0x50, 0x55, 0xc6, 0xcb, 0x50, 0xfc, 0x56, 0x20, 0xaa, 0x8e, 0x4a, 0xfa, + 0x50, 0x85, 0xbc, 0xf6, 0x0d, 0x54, 0xd2, 0x0f, 0x08, 0x54, 0x85, 0xd2, 0x61, 0xff, 0xf3, 0xfe, + 0x83, 0xa3, 0x7e, 0x2d, 0x87, 0x2a, 0x60, 0x3c, 0x3c, 0xec, 0xe1, 0xaf, 0x6a, 0x1a, 0x2a, 0x43, + 0x01, 0x1f, 0xde, 0xef, 0xd5, 0xf2, 0x5c, 0x63, 0xb0, 0x77, 0xa7, 0xb7, 0xb3, 0x85, 0x6b, 0x3a, + 0xd7, 0x18, 0x1c, 0x3c, 0xc0, 0xbd, 0x5a, 0x81, 0xe3, 0xb8, 0xb7, 0xd3, 0xdb, 0x7b, 0xd4, 0xab, + 0x19, 0x1c, 0xbf, 0xd3, 0xdb, 0x3e, 0xbc, 0x57, 0x2b, 0x5e, 0xdb, 0x86, 0x02, 0xff, 0x05, 0x46, + 0x25, 0xd0, 0xf1, 0xd6, 0x91, 0xb4, 0xba, 0xf3, 0xe0, 0xb0, 0x7f, 0x50, 0xd3, 0x38, 0x36, 0x38, + 0xdc, 0xaf, 0xe5, 0xf9, 0x61, 0x7f, 0xaf, 0x5f, 0xd3, 0xc5, 0x61, 0xeb, 0x4b, 0x69, 0x4e, 0x68, + 0xf5, 0x70, 0xcd, 0xe8, 0x7e, 0x97, 0x07, 0x43, 0xf8, 0x88, 0x3e, 0x86, 0x82, 0x58, 0xcd, 0x17, + 0x93, 0x8c, 0x2e, 0x7d, 0xcf, 0x35, 0xea, 0x59, 0x50, 0xe5, 0xef, 0x53, 0x28, 0xca, 0xfd, 0x85, + 0x2e, 0x65, 0xf7, 0x59, 0x72, 0xed, 0xf2, 0x59, 0x58, 0x5e, 0xbc, 0xa1, 0xa1, 0x1d, 0x80, 0xc5, + 0x5c, 0xa1, 0xf5, 0x4c, 0x15, 0x97, 0xb7, 0x54, 0xa3, 0x71, 0x1e, 0xa5, 0xde, 0xbf, 0x0b, 0xd5, + 0xa5, 0xb2, 0xa2, 0xac, 0x6a, 0x66, 0x78, 0x1a, 0x57, 0xcf, 0xe5, 0xa4, 0x9d, 0x6e, 0x1f, 0xd6, + 0xc4, 0x17, 0x34, 0x9f, 0x0a, 0x99, 0x8c, 0xdb, 0x50, 0xc5, 0x64, 0x16, 0x30, 0x22, 0x70, 0x94, + 0x86, 0xbf, 0xfc, 0xa1, 0xdd, 0xb8, 0x74, 0x06, 0x55, 0x1f, 0xe4, 0xb9, 0xed, 0xff, 0x3f, 0xfb, + 0xab, 0x99, 0x7b, 0xf6, 0xb2, 0xa9, 0x3d, 0x7f, 0xd9, 0xd4, 0xfe, 0x7c, 0xd9, 0xd4, 0x7e, 0x7c, + 0xd5, 0xcc, 0x3d, 0x7f, 0xd5, 0xcc, 0xfd, 0xfe, 0xaa, 0x99, 0x7b, 0x5c, 0x52, 0xff, 0x13, 0x8c, + 0x8a, 0xa2, 0x67, 0x6e, 0xfe, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x02, 0x42, 0x0e, 0xd0, 0x7d, 0x0c, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1358,6 +1412,18 @@ func (m *SeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ShardInfo != nil { + { + size, err := m.ShardInfo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintRpc(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x6a + } if m.QueryHints != nil { { size, err := m.QueryHints.MarshalToSizedBuffer(dAtA[:i]) @@ -1418,20 +1484,20 @@ func (m *SeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x30 } if len(m.Aggregates) > 0 { - dAtA4 := make([]byte, len(m.Aggregates)*10) - var j3 int + dAtA5 := make([]byte, len(m.Aggregates)*10) + var j4 int for _, num := range m.Aggregates { for num >= 1<<7 { - dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + dAtA5[j4] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j3++ + j4++ } - dAtA4[j3] = uint8(num) - j3++ + dAtA5[j4] = uint8(num) + j4++ } - i -= j3 - copy(dAtA[i:], dAtA4[:j3]) - i = encodeVarintRpc(dAtA, i, uint64(j3)) + i -= j4 + copy(dAtA[i:], dAtA5[:j4]) + i = encodeVarintRpc(dAtA, i, uint64(j4)) i-- dAtA[i] = 0x2a } @@ -1531,6 +1597,58 @@ func (m *QueryHints) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ShardInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardInfo) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Labels) > 0 { + for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Labels[iNdEx]) + copy(dAtA[i:], m.Labels[iNdEx]) + i = encodeVarintRpc(dAtA, i, uint64(len(m.Labels[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if m.By { + i-- + if m.By { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } + if m.TotalShards != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.TotalShards)) + i-- + dAtA[i] = 0x10 + } + if m.ShardIndex != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.ShardIndex)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *Func) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2112,6 +2230,10 @@ func (m *SeriesRequest) Size() (n int) { l = m.QueryHints.Size() n += 1 + l + sovRpc(uint64(l)) } + if m.ShardInfo != nil { + l = m.ShardInfo.Size() + n += 1 + l + sovRpc(uint64(l)) + } return n } @@ -2139,6 +2261,30 @@ func (m *QueryHints) Size() (n int) { return n } +func (m *ShardInfo) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ShardIndex != 0 { + n += 1 + sovRpc(uint64(m.ShardIndex)) + } + if m.TotalShards != 0 { + n += 1 + sovRpc(uint64(m.TotalShards)) + } + if m.By { + n += 2 + } + if len(m.Labels) > 0 { + for _, s := range m.Labels { + l = len(s) + n += 1 + l + sovRpc(uint64(l)) + } + } + return n +} + func (m *Func) Size() (n int) { if m == nil { return 0 @@ -3118,6 +3264,42 @@ func (m *SeriesRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 13: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ShardInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRpc + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ShardInfo == nil { + m.ShardInfo = &ShardInfo{} + } + if err := m.ShardInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -3316,6 +3498,146 @@ func (m *QueryHints) Unmarshal(dAtA []byte) error { } return nil } +func (m *ShardInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ShardIndex", wireType) + } + m.ShardIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ShardIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalShards", wireType) + } + m.TotalShards = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalShards |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field By", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.By = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRpc + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Func) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/store/storepb/rpc.proto b/pkg/store/storepb/rpc.proto index 71f004463b..72afaba8ed 100644 --- a/pkg/store/storepb/rpc.proto +++ b/pkg/store/storepb/rpc.proto @@ -118,8 +118,13 @@ message SeriesRequest { // query_hints are the hints coming from the PromQL engine when // requesting a storage.SeriesSet for a given expression. QueryHints query_hints = 12; + + // shard_info is used by the querier to request a specific + // shard of blocks instead of entire blocks. + ShardInfo shard_info = 13; } + // Analogous to storage.SelectHints. message QueryHints { // Query step size in milliseconds. @@ -135,6 +140,21 @@ message QueryHints { Range range = 5; } +// ShardInfo are the parameters used to shard series in Stores. +message ShardInfo { + // The index of the current shard. + int64 shard_index = 1; + + // The total number of shards. + int64 total_shards = 2; + + // Group by or without labels. + bool by = 3; + + // Labels on which to partition series. + repeated string labels = 4; +} + message Func { // The function or aggregation name string name = 1; diff --git a/pkg/store/storepb/shard_info.go b/pkg/store/storepb/shard_info.go new file mode 100644 index 0000000000..633751c1f7 --- /dev/null +++ b/pkg/store/storepb/shard_info.go @@ -0,0 +1,104 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storepb + +import ( + "github.com/alecthomas/units" + "github.com/cespare/xxhash/v2" + "github.com/prometheus/prometheus/model/labels" + "github.com/thanos-io/thanos/pkg/store/labelpb" +) + +var sep = []byte{'\xff'} + +type ShardMatcher struct { + buf []byte + shardingLabelset map[string]struct{} + + isSharded bool + + by bool + totalShards int64 + shardIndex int64 +} + +func (s *ShardMatcher) IsSharded() bool { + return s.isSharded +} + +func (s *ShardMatcher) MatchesZLabels(zLabels []labelpb.ZLabel) bool { + // Match all series when query is not sharded + if s == nil || !s.isSharded { + return true + } + + s.buf = s.buf[:0] + for _, lbl := range zLabels { + // Exclude metric name and le label from sharding + if lbl.Name == "__name__" || lbl.Name == "le" { + continue + } + + if shardByLabel(s.shardingLabelset, lbl, s.by) { + s.buf = append(s.buf, lbl.Name...) + s.buf = append(s.buf, sep[0]) + s.buf = append(s.buf, lbl.Value...) + s.buf = append(s.buf, sep[0]) + } + } + + hash := xxhash.Sum64(s.buf) + return hash%uint64(s.totalShards) == uint64(s.shardIndex) +} + +func (s *ShardMatcher) MatchesLabels(lbls labels.Labels) bool { + return s.MatchesZLabels(labelpb.ZLabelsFromPromLabels(lbls)) +} + +func shardByLabel(labelSet map[string]struct{}, zlabel labelpb.ZLabel, groupingBy bool) bool { + _, shardHasLabel := labelSet[zlabel.Name] + if groupingBy && shardHasLabel { + return true + } + + groupingWithout := !groupingBy + if groupingWithout && !shardHasLabel { + return true + } + + return false +} + +func (m *ShardInfo) Matcher() *ShardMatcher { + if m == nil || m.TotalShards < 1 { + return &ShardMatcher{ + isSharded: false, + } + } + + return &ShardMatcher{ + isSharded: true, + buf: make([]byte, 10*units.Kilobyte), + shardingLabelset: m.labelSet(), + by: m.By, + totalShards: m.TotalShards, + shardIndex: m.ShardIndex, + } +} + +func (m *ShardInfo) labelSet() map[string]struct{} { + if m == nil { + return nil + } + labelSet := make(map[string]struct{}) + if m == nil || m.Labels == nil { + return labelSet + } + + for _, label := range m.Labels { + labelSet[label] = struct{}{} + } + + return labelSet +} diff --git a/pkg/store/storepb/shard_info_test.go b/pkg/store/storepb/shard_info_test.go new file mode 100644 index 0000000000..957e475975 --- /dev/null +++ b/pkg/store/storepb/shard_info_test.go @@ -0,0 +1,119 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storepb + +import ( + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/thanos-io/thanos/pkg/store/labelpb" +) + +func TestShardInfo_MatchesSeries(t *testing.T) { + series := labelpb.ZLabelsFromPromLabels(labels.FromStrings( + "pod", "nginx", + "node", "node-1", + "container", "nginx", + )) + + tests := []struct { + name string + shardInfo *ShardInfo + series []labelpb.ZLabel + matches bool + }{ + { + name: "nil shard info", + shardInfo: nil, + matches: true, + }, + { + name: "one shard only", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 0, + TotalShards: 1, + By: true, + Labels: nil, + }, + matches: true, + }, + { + name: "shard by empty sharding labels", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 0, + TotalShards: 2, + By: true, + Labels: nil, + }, + matches: false, + }, + { + name: "shard without empty sharding labels", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 0, + TotalShards: 2, + By: false, + Labels: nil, + }, + matches: true, + }, + { + name: "shard by labels for shard 0", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 0, + TotalShards: 2, + By: true, + Labels: []string{"pod", "node"}, + }, + matches: false, + }, + { + name: "shard by labels for shard 1", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 1, + TotalShards: 2, + By: true, + Labels: []string{"pod", "node"}, + }, + matches: true, + }, + { + name: "shard without labels for shard 0", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 0, + TotalShards: 2, + By: false, + Labels: []string{"node"}, + }, + matches: true, + }, + { + name: "shard without labels for shard 1", + series: series, + shardInfo: &ShardInfo{ + ShardIndex: 1, + TotalShards: 2, + By: false, + Labels: []string{"node"}, + }, + matches: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matcher := test.shardInfo.Matcher() + isMatch := matcher.MatchesZLabels(test.series) + if isMatch != test.matches { + t.Fatalf("invalid result, got %t, want %t", isMatch, test.matches) + } + }) + } +} diff --git a/pkg/store/storepb/types.proto b/pkg/store/storepb/types.proto index 2cf255713b..d80e1af1ba 100644 --- a/pkg/store/storepb/types.proto +++ b/pkg/store/storepb/types.proto @@ -70,4 +70,4 @@ enum PartialResponseStrategy { /// This is especially useful for any rule/alert evaluations on top of StoreAPI which usually does not tolerate partial /// errors. ABORT = 1; -} \ No newline at end of file +} diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index c13391a691..19654abee0 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -152,9 +152,14 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer set := q.Select(false, nil, matchers...) + shardMatcher := r.ShardInfo.Matcher() // Stream at most one series per frame; series may be split over multiple frames according to maxBytesInFrame. for set.Next() { series := set.At() + if !shardMatcher.MatchesLabels(series.Labels()) { + continue + } + storeSeries := storepb.Series{Labels: labelpb.ZLabelsFromPromLabels(labelpb.ExtendSortedLabels(series.Labels(), s.extLset))} if r.SkipChunks { if err := srv.Send(storepb.NewSeriesResponse(&storeSeries)); err != nil { diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 7a354f41a3..c8051d3878 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -719,26 +719,34 @@ func (c *CompactorBuilder) Init(bucketConfig client.BucketConfig, relabelConfig })) } -func NewQueryFrontend(e e2e.Environment, name, downstreamURL string, cacheConfig queryfrontend.CacheProviderConfig) e2e.InstrumentedRunnable { +func NewQueryFrontend(e e2e.Environment, name, downstreamURL string, config queryfrontend.Config, cacheConfig queryfrontend.CacheProviderConfig) e2e.InstrumentedRunnable { cacheConfigBytes, err := yaml.Marshal(cacheConfig) if err != nil { return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "marshal response cache config file: %v", cacheConfig)) } - args := e2e.BuildArgs(map[string]string{ + flags := map[string]string{ "--debug.name": fmt.Sprintf("query-frontend-%s", name), "--http-address": ":8080", "--query-frontend.downstream-url": downstreamURL, "--log.level": infoLogLevel, "--query-range.response-cache-config": string(cacheConfigBytes), - }) + } + + if !config.QueryRangeConfig.AlignRangeWithStep { + flags["--no-query-range.align-range-with-step"] = "" + } + + if config.NumShards > 0 { + flags["--query-frontend.num-shards"] = strconv.Itoa(config.NumShards) + } return e2e.NewInstrumentedRunnable( e, fmt.Sprintf("query-frontend-%s", name), ).WithPorts(map[string]int{"http": 8080}, "http").Init( e2e.StartOptions{ Image: DefaultImage(), - Command: e2e.NewCommand("query-frontend", args...), + Command: e2e.NewCommand("query-frontend", e2e.BuildArgs(flags)...), Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), User: strconv.Itoa(os.Getuid()), WaitReadyBackoff: &defaultBackoffConfig, diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index df9fec28d2..668ae71013 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -9,6 +9,9 @@ import ( "testing" "time" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" + "github.com/efficientgo/e2e" "github.com/efficientgo/e2e/matchers" "github.com/pkg/errors" @@ -46,7 +49,8 @@ func TestQueryFrontend(t *testing.T) { }, } - queryFrontend := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), inMemoryCacheConfig) + cfg := queryfrontend.Config{} + queryFrontend := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), cfg, inMemoryCacheConfig) testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend)) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) @@ -424,7 +428,8 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { }, } - queryFrontend := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), memCachedConfig) + cfg := queryfrontend.Config{} + queryFrontend := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), cfg, memCachedConfig) testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend)) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) @@ -515,3 +520,73 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(2), "cortex_cache_fetched_keys_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "cortex_cache_hits_total")) } + +func TestRangeQueryShardingWithRandomData(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("e2e_test_range_query_sharding_random_data") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + promConfig := e2ethanos.DefaultPromConfig("p1", 0, "", "") + prom, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "p1", promConfig, "", e2ethanos.DefaultPrometheusImage(), "", "remote-write-receiver") + + now := model.Now() + ctx := context.Background() + timeSeries := []labels.Labels{ + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "1"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "1"}, {Name: "handler", Value: "/metrics"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "2"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "2"}, {Name: "handler", Value: "/metrics"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "3"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "3"}, {Name: "handler", Value: "/metrics"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "4"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "4"}, {Name: "handler", Value: "/metrics"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "5"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "5"}, {Name: "handler", Value: "/metrics"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "6"}, {Name: "handler", Value: "/"}}, + {{Name: labels.MetricName, Value: "http_requests_total"}, {Name: "pod", Value: "6"}, {Name: "handler", Value: "/metrics"}}, + } + + startTime := now.Time().Add(-1 * time.Hour) + endTime := now.Time().Add(1 * time.Hour) + _, err = e2eutil.CreateBlock(ctx, prom.Dir(), timeSeries, 20, timestamp.FromTime(startTime), timestamp.FromTime(endTime), nil, 0, metadata.NoneFunc) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) + + stores := []string{sidecar.InternalEndpoint("grpc")} + q1 := e2ethanos.NewQuerierBuilder(e, "q1", stores...).Init() + testutil.Ok(t, e2e.StartAndWaitReady(q1)) + + inMemoryCacheConfig := queryfrontend.CacheProviderConfig{ + Type: queryfrontend.INMEMORY, + Config: queryfrontend.InMemoryResponseCacheConfig{ + MaxSizeItems: 1000, + Validity: time.Hour, + }, + } + config := queryfrontend.Config{ + QueryRangeConfig: queryfrontend.QueryRangeConfig{ + AlignRangeWithStep: false, + }, + NumShards: 2, + } + qfe := e2ethanos.NewQueryFrontend(e, "query-frontend", "http://"+q1.InternalEndpoint("http"), config, inMemoryCacheConfig) + testutil.Ok(t, e2e.StartAndWaitReady(qfe)) + + qryFunc := func() string { return `sum by (pod) (http_requests_total)` } + queryOpts := promclient.QueryOptions{Deduplicate: true} + + var resultWithoutSharding model.Matrix + rangeQuery(t, ctx, q1.Endpoint("http"), qryFunc, timestamp.FromTime(startTime), timestamp.FromTime(endTime), 30, queryOpts, func(res model.Matrix) error { + resultWithoutSharding = res + return nil + }) + var resultWithSharding model.Matrix + rangeQuery(t, ctx, qfe.Endpoint("http"), qryFunc, timestamp.FromTime(startTime), timestamp.FromTime(endTime), 30, queryOpts, func(res model.Matrix) error { + resultWithSharding = res + return nil + }) + + testutil.Equals(t, resultWithoutSharding, resultWithSharding) +} diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 6090cba2e9..c996501f2a 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -624,7 +624,7 @@ func TestSidecarStorePushdown(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(q)) testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(1), "thanos_blocks_meta_synced")) - testutil.Ok(t, synthesizeSamples(ctx, prom1, []fakeMetricSample{ + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom1, []fakeMetricSample{ { label: "foo", value: 123, @@ -814,8 +814,8 @@ func TestSidecarQueryEvaluation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, synthesizeSamples(ctx, prom1, tc.prom1Samples)) - testutil.Ok(t, synthesizeSamples(ctx, prom2, tc.prom2Samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom1, tc.prom1Samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom2, tc.prom2Samples)) testQuery := func() string { return tc.query } queryAndAssert(t, ctx, q.Endpoint("http"), testQuery, time.Now, promclient.QueryOptions{ @@ -1011,12 +1011,16 @@ func queryExemplars(t *testing.T, ctx context.Context, addr, q string, start, en })) } -func synthesizeSamples(ctx context.Context, prometheus e2e.InstrumentedRunnable, testSamples []fakeMetricSample) error { +func synthesizeFakeMetricSamples(ctx context.Context, prometheus e2e.InstrumentedRunnable, testSamples []fakeMetricSample) error { samples := make([]model.Sample, len(testSamples)) for i, s := range testSamples { samples[i] = newSample(s) } + return synthesizeSamples(ctx, prometheus, samples) +} + +func synthesizeSamples(ctx context.Context, prometheus e2e.InstrumentedRunnable, samples []model.Sample) error { remoteWriteURL, err := url.Parse("http://" + prometheus.Endpoint("http") + "/api/v1/write") if err != nil { return err @@ -1182,8 +1186,8 @@ func TestSidecarQueryEvaluationWithDedup(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, synthesizeSamples(ctx, prom1, tc.prom1Samples)) - testutil.Ok(t, synthesizeSamples(ctx, prom2, tc.prom2Samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom1, tc.prom1Samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom2, tc.prom2Samples)) testQuery := func() string { return tc.query } queryAndAssert(t, ctx, q.Endpoint("http"), testQuery, time.Now, promclient.QueryOptions{ @@ -1234,7 +1238,7 @@ func TestSidecarAlignmentPushdown(t *testing.T) { }) } - testutil.Ok(t, synthesizeSamples(ctx, prom1, samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom1, samples)) // This query should have identical requests. testQuery := func() string { return `max_over_time({instance="test"}[5m])` } @@ -1309,7 +1313,7 @@ func TestGrpcInstantQuery(t *testing.T) { }, } ctx := context.Background() - testutil.Ok(t, synthesizeSamples(ctx, prom, samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom, samples)) grpcConn, err := grpc.Dial(querier.Endpoint("grpc"), grpc.WithTransportCredentials(insecure.NewCredentials())) testutil.Ok(t, err) @@ -1431,7 +1435,7 @@ func TestGrpcQueryRange(t *testing.T) { }, } ctx := context.Background() - testutil.Ok(t, synthesizeSamples(ctx, prom, samples)) + testutil.Ok(t, synthesizeFakeMetricSamples(ctx, prom, samples)) grpcConn, err := grpc.Dial(querier.Endpoint("grpc"), grpc.WithTransportCredentials(insecure.NewCredentials())) testutil.Ok(t, err) From 2ec5aae6a74b4fee9961c6de65e279fd64b58ef7 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 13 Jul 2022 19:23:17 +0200 Subject: [PATCH 3/6] First code review pass Signed-off-by: Filip Petkovski --- pkg/querysharding/analysis.go | 8 ++++---- pkg/querysharding/analyzer.go | 2 +- pkg/store/prometheus.go | 1 - test/e2e/query_frontend_test.go | 5 ++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/querysharding/analysis.go b/pkg/querysharding/analysis.go index 90f9fcc806..d46a0048c6 100644 --- a/pkg/querysharding/analysis.go +++ b/pkg/querysharding/analysis.go @@ -29,7 +29,7 @@ func newShardableByLabels(labels []string, by bool) QueryAnalysis { } } -func (q QueryAnalysis) scopeToLabels(labels []string, by bool) QueryAnalysis { +func (q *QueryAnalysis) scopeToLabels(labels []string, by bool) QueryAnalysis { labels = without(labels, excludedLabels) if q.shardingLabels == nil { @@ -52,11 +52,11 @@ func (q QueryAnalysis) scopeToLabels(labels []string, by bool) QueryAnalysis { } } -func (q QueryAnalysis) IsShardable() bool { +func (q *QueryAnalysis) IsShardable() bool { return len(q.shardingLabels) > 0 } -func (q QueryAnalysis) ShardingLabels() []string { +func (q *QueryAnalysis) ShardingLabels() []string { if len(q.shardingLabels) == 0 { return nil } @@ -64,7 +64,7 @@ func (q QueryAnalysis) ShardingLabels() []string { return q.shardingLabels } -func (q QueryAnalysis) ShardBy() bool { +func (q *QueryAnalysis) ShardBy() bool { return q.shardBy } diff --git a/pkg/querysharding/analyzer.go b/pkg/querysharding/analyzer.go index e3fe99f951..b0e5538efa 100644 --- a/pkg/querysharding/analyzer.go +++ b/pkg/querysharding/analyzer.go @@ -27,7 +27,7 @@ func NewQueryAnalyzer() *QueryAnalyzer { // Analyze uses the following algorithm: // * if a query has subqueries, such as label_join or label_replace, -// then treat the query as non shardable. +// or has functions which cannot be sharded, then treat the query as non shardable. // * if the query's root expression has grouping labels, // then treat the query as shardable by those labels. // * if the query's root expression has no grouping labels, diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 93991e06a2..8fbfa12d0f 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -201,7 +201,6 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie } q := &prompb.Query{StartTimestampMs: r.MinTime, EndTimestampMs: r.MaxTime} - level.Info(p.logger).Log("min_time", r.MinTime, "max_time", r.MaxTime) for _, m := range matchers { pm := &prompb.LabelMatcher{Name: m.Name, Value: m.Value} diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 668ae71013..7a4cbf5d7a 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -9,9 +9,6 @@ import ( "testing" "time" - "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/thanos-io/thanos/pkg/testutil/e2eutil" - "github.com/efficientgo/e2e" "github.com/efficientgo/e2e/matchers" "github.com/pkg/errors" @@ -19,10 +16,12 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" + "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/cacheutil" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/queryfrontend" "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) From bdc09237bedb5d0b2d7cb4470f737b80c745ddbe Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 19 Jul 2022 09:53:09 +0200 Subject: [PATCH 4/6] Use sync pool to reuse sharding buffers Signed-off-by: Filip Petkovski --- pkg/queryfrontend/shard_query.go | 3 ++- pkg/store/bucket.go | 19 +++++++++++------ pkg/store/prometheus.go | 4 +++- pkg/store/proxy.go | 18 +++++++++++----- pkg/store/storepb/shard_info.go | 32 +++++++++++++++++----------- pkg/store/storepb/shard_info_test.go | 10 ++++++++- pkg/store/tsdb.go | 9 +++++++- 7 files changed, 68 insertions(+), 27 deletions(-) diff --git a/pkg/queryfrontend/shard_query.go b/pkg/queryfrontend/shard_query.go index e2bab7a94b..89f97e569f 100644 --- a/pkg/queryfrontend/shard_query.go +++ b/pkg/queryfrontend/shard_query.go @@ -9,7 +9,8 @@ package queryfrontend import ( "context" - "github.com/cortexproject/cortex/pkg/querier/queryrange" + "github.com/thanos-io/thanos/internal/cortex/querier/queryrange" + "github.com/thanos-io/thanos/pkg/querysharding" "github.com/thanos-io/thanos/pkg/store/storepb" ) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 64f3163341..4732d36cf0 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -279,6 +279,7 @@ type BucketStore struct { dir string indexCache storecache.IndexCache indexReaderPool *indexheader.ReaderPool + buffers sync.Pool chunkPool pool.Bytes // Sets of blocks that have the same labels. They are indexed by a hash over their label set. @@ -401,11 +402,15 @@ func NewBucketStore( options ...BucketStoreOption, ) (*BucketStore, error) { s := &BucketStore{ - logger: log.NewNopLogger(), - bkt: bkt, - fetcher: fetcher, - dir: dir, - indexCache: noopCache{}, + logger: log.NewNopLogger(), + bkt: bkt, + fetcher: fetcher, + dir: dir, + indexCache: noopCache{}, + buffers: sync.Pool{New: func() interface{} { + b := make([]byte, 0, initialBufSize) + return &b + }}, chunkPool: pool.NoopBytes{}, blocks: map[ulid.ULID]*bucketBlock{}, blockSets: map[uint64]*bucketBlockSet{}, @@ -1067,6 +1072,8 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie }) defer span.Finish() + shardMatcher := req.ShardInfo.Matcher(&s.buffers) + defer shardMatcher.Close() part, pstats, err := blockSeries( newCtx, b.extLset, @@ -1078,7 +1085,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie req.SkipChunks, req.MinTime, req.MaxTime, req.Aggregates, - req.ShardInfo.Matcher(), + shardMatcher, ) if err != nil { return errors.Wrapf(err, "fetch series for block %s", b.meta.ULID) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 8fbfa12d0f..a2cd3c1dd7 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -195,7 +195,9 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie return nil } - shardMatcher := r.ShardInfo.Matcher() + shardMatcher := r.ShardInfo.Matcher(&p.buffers) + defer shardMatcher.Close() + if r.QueryHints != nil && r.QueryHints.IsSafeToExecute() && !shardMatcher.IsSharded() { return p.queryPrometheus(s, r) } diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 97a92f531f..574c3740ef 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -62,6 +62,7 @@ type ProxyStore struct { stores func() []Client component component.StoreAPI selectorLabels labels.Labels + buffers sync.Pool responseTimeout time.Duration metrics *proxyStoreMetrics @@ -104,10 +105,14 @@ func NewProxyStore( metrics := newProxyStoreMetrics(reg) s := &ProxyStore{ - logger: logger, - stores: stores, - component: component, - selectorLabels: selectorLabels, + logger: logger, + stores: stores, + component: component, + selectorLabels: selectorLabels, + buffers: sync.Pool{New: func() interface{} { + b := make([]byte, 0, initialBufSize) + return &b + }}, responseTimeout: responseTimeout, metrics: metrics, } @@ -344,6 +349,7 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe s.responseTimeout, s.metrics.emptyStreamResponses, st.SupportsSharding(), + &s.buffers, r.ShardInfo, )) } @@ -441,6 +447,7 @@ func startStreamSeriesSet( responseTimeout time.Duration, emptyStreamResponses prometheus.Counter, storeSupportsSharding bool, + buffers *sync.Pool, shardInfo *storepb.ShardInfo, ) *streamSeriesSet { s := &streamSeriesSet{ @@ -491,7 +498,8 @@ func startStreamSeriesSet( } }() - shardMatcher := shardInfo.Matcher() + shardMatcher := shardInfo.Matcher(buffers) + defer shardMatcher.Close() applySharding := shardInfo != nil && !storeSupportsSharding if applySharding { msg := "Applying series sharding in the proxy since there is not support in the underlying store" diff --git a/pkg/store/storepb/shard_info.go b/pkg/store/storepb/shard_info.go index 633751c1f7..a7876429b9 100644 --- a/pkg/store/storepb/shard_info.go +++ b/pkg/store/storepb/shard_info.go @@ -4,7 +4,8 @@ package storepb import ( - "github.com/alecthomas/units" + "sync" + "github.com/cespare/xxhash/v2" "github.com/prometheus/prometheus/model/labels" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -13,11 +14,11 @@ import ( var sep = []byte{'\xff'} type ShardMatcher struct { - buf []byte + buf *[]byte + buffers *sync.Pool shardingLabelset map[string]struct{} - isSharded bool - + isSharded bool by bool totalShards int64 shardIndex int64 @@ -27,13 +28,19 @@ func (s *ShardMatcher) IsSharded() bool { return s.isSharded } +func (s *ShardMatcher) Close() { + if s.buffers != nil { + s.buffers.Put(s.buf) + } +} + func (s *ShardMatcher) MatchesZLabels(zLabels []labelpb.ZLabel) bool { // Match all series when query is not sharded if s == nil || !s.isSharded { return true } - s.buf = s.buf[:0] + *s.buf = (*s.buf)[:0] for _, lbl := range zLabels { // Exclude metric name and le label from sharding if lbl.Name == "__name__" || lbl.Name == "le" { @@ -41,14 +48,14 @@ func (s *ShardMatcher) MatchesZLabels(zLabels []labelpb.ZLabel) bool { } if shardByLabel(s.shardingLabelset, lbl, s.by) { - s.buf = append(s.buf, lbl.Name...) - s.buf = append(s.buf, sep[0]) - s.buf = append(s.buf, lbl.Value...) - s.buf = append(s.buf, sep[0]) + *s.buf = append(*s.buf, lbl.Name...) + *s.buf = append(*s.buf, sep[0]) + *s.buf = append(*s.buf, lbl.Value...) + *s.buf = append(*s.buf, sep[0]) } } - hash := xxhash.Sum64(s.buf) + hash := xxhash.Sum64(*s.buf) return hash%uint64(s.totalShards) == uint64(s.shardIndex) } @@ -70,7 +77,7 @@ func shardByLabel(labelSet map[string]struct{}, zlabel labelpb.ZLabel, groupingB return false } -func (m *ShardInfo) Matcher() *ShardMatcher { +func (m *ShardInfo) Matcher(buffers *sync.Pool) *ShardMatcher { if m == nil || m.TotalShards < 1 { return &ShardMatcher{ isSharded: false, @@ -79,7 +86,8 @@ func (m *ShardInfo) Matcher() *ShardMatcher { return &ShardMatcher{ isSharded: true, - buf: make([]byte, 10*units.Kilobyte), + buf: buffers.Get().(*[]byte), + buffers: buffers, shardingLabelset: m.labelSet(), by: m.By, totalShards: m.TotalShards, diff --git a/pkg/store/storepb/shard_info_test.go b/pkg/store/storepb/shard_info_test.go index 957e475975..ff1ef0972a 100644 --- a/pkg/store/storepb/shard_info_test.go +++ b/pkg/store/storepb/shard_info_test.go @@ -4,8 +4,11 @@ package storepb import ( + "sync" "testing" + "github.com/alecthomas/units" + "github.com/prometheus/prometheus/model/labels" "github.com/thanos-io/thanos/pkg/store/labelpb" ) @@ -107,9 +110,14 @@ func TestShardInfo_MatchesSeries(t *testing.T) { }, } + buffers := sync.Pool{New: func() interface{} { + b := make([]byte, 0, 10*units.Kilobyte) + return &b + }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { - matcher := test.shardInfo.Matcher() + matcher := test.shardInfo.Matcher(&buffers) + defer matcher.Close() isMatch := matcher.MatchesZLabels(test.series) if isMatch != test.matches { t.Fatalf("invalid result, got %t, want %t", isMatch, test.matches) diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index 19654abee0..4dcfef4882 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -8,6 +8,7 @@ import ( "io" "math" "sort" + "sync" "github.com/go-kit/log" "github.com/pkg/errors" @@ -38,6 +39,7 @@ type TSDBStore struct { db TSDBReader component component.StoreAPI extLset labels.Labels + buffers sync.Pool maxBytesPerFrame int } @@ -65,6 +67,10 @@ func NewTSDBStore(logger log.Logger, db TSDBReader, component component.StoreAPI component: component, extLset: extLset, maxBytesPerFrame: RemoteReadFrameLimit, + buffers: sync.Pool{New: func() interface{} { + b := make([]byte, 0, initialBufSize) + return &b + }}, } } @@ -152,7 +158,8 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer set := q.Select(false, nil, matchers...) - shardMatcher := r.ShardInfo.Matcher() + shardMatcher := r.ShardInfo.Matcher(&s.buffers) + defer shardMatcher.Close() // Stream at most one series per frame; series may be split over multiple frames according to maxBytesInFrame. for set.Next() { series := set.At() From ffdb6156fd40fbb1ddc262e838440b606d0c6bd1 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 20 Jul 2022 12:58:43 +0200 Subject: [PATCH 5/6] Add test for binary expression with constant Signed-off-by: Filip Petkovski --- cmd/thanos/query_frontend.go | 2 +- docs/components/query-frontend.md | 9 ++++++--- pkg/querysharding/analyzer.go | 4 +++- pkg/querysharding/analyzer_test.go | 4 ++++ test/e2e/e2ethanos/services.go | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 5857a4a3d0..e9eee5966a 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -135,7 +135,7 @@ func registerQueryFrontend(app *extkingpin.App) { cmd.Flag("query-frontend.forward-header", "List of headers forwarded by the query-frontend to downstream queriers, default is empty").PlaceHolder("").StringsVar(&cfg.ForwardHeaders) - cmd.Flag("query-frontend.num-shards", "Number of queriers to use when sharding PromQL queries").IntVar(&cfg.NumShards) + cmd.Flag("query-frontend.vertical-shards", "Number of shards to use when distributing shardable PromQL queries. For more details, you can refer to the Vertical query sharding proposal: https://thanos.io/tip/proposals-accepted/202205-vertical-query-sharding.md").IntVar(&cfg.NumShards) cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("").EnumVar(&cfg.RequestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index fd66e70837..044f87ce76 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -261,9 +261,6 @@ Flags: Log queries that are slower than the specified duration. Set to 0 to disable. Set to < 0 to enable on all queries. - --query-frontend.num-shards=QUERY-FRONTEND.NUM-SHARDS - Number of queriers to use when sharding PromQL - queries --query-frontend.org-id-header= ... Request header names used to identify the source of slow queries (repeated flag). The @@ -272,6 +269,12 @@ Flags: headers match the request, the first matching arg specified will take precedence. If no headers match 'anonymous' will be used. + --query-frontend.vertical-shards=QUERY-FRONTEND.VERTICAL-SHARDS + Number of shards to use when distributing + shardable PromQL queries. For more details, you + can refer to the Vertical query sharding + proposal: + https://thanos.io/tip/proposals-accepted/202205-vertical-query-sharding.md --query-range.align-range-with-step Mutate incoming queries to align their start and end with their step for better diff --git a/pkg/querysharding/analyzer.go b/pkg/querysharding/analyzer.go index b0e5538efa..233f3b74c5 100644 --- a/pkg/querysharding/analyzer.go +++ b/pkg/querysharding/analyzer.go @@ -54,7 +54,9 @@ func (a *QueryAnalyzer) Analyze(query string) (QueryAnalysis, error) { return fmt.Errorf("expressions with %s are not shardable", n.Func.Name) } case *parser.BinaryExpr: - analysis = analysis.scopeToLabels(n.VectorMatching.MatchingLabels, n.VectorMatching.On) + if n.VectorMatching != nil { + analysis = analysis.scopeToLabels(n.VectorMatching.MatchingLabels, n.VectorMatching.On) + } case *parser.AggregateExpr: labels := make([]string, 0) if len(n.Grouping) > 0 { diff --git a/pkg/querysharding/analyzer_test.go b/pkg/querysharding/analyzer_test.go index 9dbb9c948d..d34e7b29e4 100644 --- a/pkg/querysharding/analyzer_test.go +++ b/pkg/querysharding/analyzer_test.go @@ -42,6 +42,10 @@ func TestAnalyzeQuery(t *testing.T) { name: "binary expression", expression: `http_requests_total{code="400"} / http_requests_total`, }, + { + name: "binary expression with constant", + expression: `http_requests_total{code="400"} / 4`, + }, { name: "binary expression with empty vector matching", expression: `http_requests_total{code="400"} / on () http_requests_total`, diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index c8051d3878..d39055ee51 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -738,7 +738,7 @@ func NewQueryFrontend(e e2e.Environment, name, downstreamURL string, config quer } if config.NumShards > 0 { - flags["--query-frontend.num-shards"] = strconv.Itoa(config.NumShards) + flags["--query-frontend.vertical-shards"] = strconv.Itoa(config.NumShards) } return e2e.NewInstrumentedRunnable( From 6d0489b745714b975a1f288425e092dd0053b3ee Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Fri, 22 Jul 2022 10:34:59 +0200 Subject: [PATCH 6/6] Include external labels in series sharding Signed-off-by: Filip Petkovski --- pkg/store/bucket.go | 5 +++-- pkg/store/prometheus.go | 5 +++-- pkg/store/proxy.go | 9 +++++++++ pkg/store/tsdb.go | 5 +++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 4732d36cf0..7f526ce09d 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -835,12 +835,13 @@ func blockSeries( return nil, nil, errors.Wrap(err, "Lookup labels symbols") } - if !shardMatcher.MatchesLabels(lset) { + completeLabelset := labelpb.ExtendSortedLabels(lset, extLset) + if !shardMatcher.MatchesLabels(completeLabelset) { continue } s := seriesEntry{} - s.lset = labelpb.ExtendSortedLabels(lset, extLset) + s.lset = completeLabelset if !skipChunks { // Schedule loading chunks. diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index a2cd3c1dd7..50d4aa984f 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -396,7 +396,8 @@ func (p *PrometheusStore) handleStreamedPrometheusResponse( framesNum++ for _, series := range res.ChunkedSeries { - if !shardMatcher.MatchesZLabels(series.Labels) { + completeLabelset := labelpb.ExtendSortedLabels(labelpb.ZLabelsToPromLabels(series.Labels), extLset) + if !shardMatcher.MatchesLabels(completeLabelset) { continue } @@ -423,7 +424,7 @@ func (p *PrometheusStore) handleStreamedPrometheusResponse( r := storepb.NewSeriesResponse(&storepb.Series{ Labels: labelpb.ZLabelsFromPromLabels( - labelpb.ExtendSortedLabels(labelpb.ZLabelsToPromLabels(series.Labels), extLset), + completeLabelset, ), Chunks: thanosChks, }) diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 574c3740ef..54a816e396 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -468,6 +468,15 @@ func startStreamSeriesSet( bytesProcessed := 0 defer func() { + if shardInfo != nil { + level.Info(logger).Log("msg", "Done fetching series", + "series", seriesStats.Series, + "chunks", seriesStats.Chunks, + "samples", seriesStats.Samples, + "bytes", bytesProcessed, + ) + } + span.SetTag("processed.series", seriesStats.Series) span.SetTag("processed.chunks", seriesStats.Chunks) span.SetTag("processed.samples", seriesStats.Samples) diff --git a/pkg/store/tsdb.go b/pkg/store/tsdb.go index 4dcfef4882..e3f53695fd 100644 --- a/pkg/store/tsdb.go +++ b/pkg/store/tsdb.go @@ -163,11 +163,12 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSer // Stream at most one series per frame; series may be split over multiple frames according to maxBytesInFrame. for set.Next() { series := set.At() - if !shardMatcher.MatchesLabels(series.Labels()) { + completeLabelset := labelpb.ExtendSortedLabels(series.Labels(), s.extLset) + if !shardMatcher.MatchesLabels(completeLabelset) { continue } - storeSeries := storepb.Series{Labels: labelpb.ZLabelsFromPromLabels(labelpb.ExtendSortedLabels(series.Labels(), s.extLset))} + storeSeries := storepb.Series{Labels: labelpb.ZLabelsFromPromLabels(completeLabelset)} if r.SkipChunks { if err := srv.Send(storepb.NewSeriesResponse(&storeSeries)); err != nil { return status.Error(codes.Aborted, err.Error())