Skip to content

Commit

Permalink
match support for labels api in query frontend (#4585)
Browse files Browse the repository at this point in the history 
* match support for labels api in query frontend

Signed-off-by: Sandeep Raveesh <crsandy@gmail.com>

* removing the redundant log

Signed-off-by: Sandeep Raveesh <crsandy@gmail.com>

* removing the unused import

Signed-off-by: Sandeep Raveesh <crsandy@gmail.com>
  • Loading branch information
@crsandeep
crsandeep committed Aug 23, 2021
1 parent a166e93 commit 291b930
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 13 deletions.
2 changes: 1 addition & 1 deletion pkg/queryfrontend/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (t thanosCacheKeyGenerator) GenerateCacheKey(_ string, r queryrange.Request
}
return fmt.Sprintf("%s:%d:%d:%d", tr.Query, tr.Step, currentInterval, i)
case *ThanosLabelsRequest:
return fmt.Sprintf("%s:%d", tr.Label, currentInterval)
return fmt.Sprintf("%s:%s:%d", tr.Label, tr.Matchers, currentInterval)
case *ThanosSeriesRequest:
return fmt.Sprintf("%s:%d", tr.Matchers, currentInterval)
}
Expand Down
56 changes: 56 additions & 0 deletions pkg/queryfrontend/cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"testing"

"github.com/cortexproject/cortex/pkg/querier/queryrange"
"github.com/prometheus/prometheus/pkg/labels"

"github.com/thanos-io/thanos/pkg/testutil"
)
Expand Down Expand Up @@ -76,6 +77,61 @@ func TestGenerateCacheKey(t *testing.T) {
},
expected: "up:10000:0:0",
},
{
name: "label names, no matcher",
req: &ThanosLabelsRequest{
Start: 0,
},
expected: ":[]:0",
},
{
name: "label names, single matcher",
req: &ThanosLabelsRequest{
Start: 0,
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
},
expected: `:[[foo="bar"]]:0`,
},
{
name: "label names, multiple matchers",
req: &ThanosLabelsRequest{
Start: 0,
Matchers: [][]*labels.Matcher{
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")},
{labels.MustNewMatcher(labels.MatchEqual, "baz", "qux")},
},
},
expected: `:[[foo="bar"] [baz="qux"]]:0`,
},
{
name: "label values, no matcher",
req: &ThanosLabelsRequest{
Start: 0,
Label: "up",
},
expected: "up:[]:0",
},
{
name: "label values, single matcher",
req: &ThanosLabelsRequest{
Start: 0,
Label: "up",
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
},
expected: `up:[[foo="bar"]]:0`,
},
{
name: "label values, multiple matchers",
req: &ThanosLabelsRequest{
Start: 0,
Label: "up",
Matchers: [][]*labels.Matcher{
{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")},
{labels.MustNewMatcher(labels.MatchEqual, "baz", "qux")},
},
},
expected: `up:[[foo="bar"] [baz="qux"]]:0`,
},
} {
key := splitter.GenerateCacheKey("", tc.req)
testutil.Equals(t, tc.expected, key)
Expand Down
8 changes: 8 additions & 0 deletions pkg/queryfrontend/labels_codec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,9 @@ func (c labelsCodec) EncodeRequest(ctx context.Context, r queryrange.Request) (*
"end": []string{encodeTime(thanosReq.End)},
queryv1.PartialResponseParam: []string{strconv.FormatBool(thanosReq.PartialResponse)},
}
if len(thanosReq.Matchers) > 0 {
params[queryv1.MatcherParam] = matchersToStringSlice(thanosReq.Matchers)
}
if len(thanosReq.StoreMatchers) > 0 {
params[queryv1.StoreMatcherParam] = matchersToStringSlice(thanosReq.StoreMatchers)
}
Expand Down Expand Up @@ -278,6 +281,11 @@ func (c labelsCodec) parseLabelsRequest(r *http.Request, op string) (queryrange.
return nil, err
}

result.Matchers, err = parseMatchersParam(r.Form, queryv1.MatcherParam)
if err != nil {
return nil, err
}

result.PartialResponse, err = parsePartialResponseParam(r.FormValue(queryv1.PartialResponseParam), c.partialResponse)
if err != nil {
return nil, err
Expand Down
16 changes: 10 additions & 6 deletions pkg/queryfrontend/labels_codec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,26 +91,28 @@ func TestLabelsCodec_DecodeRequest(t *testing.T) {
},
{
name: "label_names partial_response default to true",
url: "/api/v1/labels?start=123&end=456",
url: `/api/v1/labels?start=123&end=456&match[]={foo="bar"}`,
partialResponse: true,
expectedRequest: &ThanosLabelsRequest{
Path: "/api/v1/labels",
Start: 123000,
End: 456000,
PartialResponse: true,
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
StoreMatchers: [][]*labels.Matcher{},
},
},
{
name: "label_values partial_response default to true",
url: "/api/v1/label/__name__/values?start=123&end=456",
url: `/api/v1/label/__name__/values?start=123&end=456&match[]={foo="bar"}`,
partialResponse: true,
expectedRequest: &ThanosLabelsRequest{
Path: "/api/v1/label/__name__/values",
Start: 123000,
End: 456000,
PartialResponse: true,
Label: "__name__",
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
StoreMatchers: [][]*labels.Matcher{},
},
},
Expand All @@ -130,13 +132,14 @@ func TestLabelsCodec_DecodeRequest(t *testing.T) {
},
{
name: "partial_response default to false, but set to true in query",
url: "/api/v1/labels?start=123&end=456&partial_response=true",
url: `/api/v1/labels?start=123&end=456&partial_response=true&match[]={foo="bar"}`,
partialResponse: false,
expectedRequest: &ThanosLabelsRequest{
Path: "/api/v1/labels",
Start: 123000,
End: 456000,
PartialResponse: true,
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
StoreMatchers: [][]*labels.Matcher{},
},
},
Expand All @@ -145,9 +148,10 @@ func TestLabelsCodec_DecodeRequest(t *testing.T) {
url: `/api/v1/labels?start=123&end=456&storeMatch[]={__address__="localhost:10901", cluster="test"}`,
partialResponse: false,
expectedRequest: &ThanosLabelsRequest{
Path: "/api/v1/labels",
Start: 123000,
End: 456000,
Path: "/api/v1/labels",
Start: 123000,
End: 456000,
Matchers: [][]*labels.Matcher{},
StoreMatchers: [][]*labels.Matcher{
{
labels.MustNewMatcher(labels.MatchEqual, "__address__", "localhost:10901"),
Expand Down
2 changes: 2 additions & 0 deletions pkg/queryfrontend/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type ThanosLabelsRequest struct {
End int64
Label string
Path string
Matchers [][]*labels.Matcher
StoreMatchers [][]*labels.Matcher
PartialResponse bool
CachingOptions queryrange.CachingOptions
Expand Down Expand Up @@ -143,6 +144,7 @@ func (r *ThanosLabelsRequest) LogToSpan(sp opentracing.Span) {
otlog.String("start", timestamp.Time(r.GetStart()).String()),
otlog.String("end", timestamp.Time(r.GetEnd()).String()),
otlog.Bool("partial_response", r.PartialResponse),
otlog.Object("matchers", r.Matchers),
otlog.Object("storeMatchers", r.StoreMatchers),
}
if r.Label != "" {
Expand Down
32 changes: 26 additions & 6 deletions pkg/queryfrontend/roundtrip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,14 @@ func TestRoundTripLabelsCacheMiddleware(t *testing.T) {
End: 2 * hour,
}

// Same query params as testRequest, but with Matchers
testRequestWithMatchers := &ThanosLabelsRequest{
Path: "/api/v1/labels",
Start: 0,
End: 2 * hour,
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
}

// Same query params as testRequest, but with storeMatchers
testRequestWithStoreMatchers := &ThanosLabelsRequest{
Path: "/api/v1/labels",
Expand All @@ -522,6 +530,14 @@ func TestRoundTripLabelsCacheMiddleware(t *testing.T) {
Label: "foo",
}

testLabelValuesRequestFooWithMatchers := &ThanosLabelsRequest{
Path: "/api/v1/label/foo/values",
Start: 0,
End: 2 * hour,
Label: "foo",
Matchers: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}},
}

testLabelValuesRequestBar := &ThanosLabelsRequest{
Path: "/api/v1/label/bar/values",
Start: 0,
Expand Down Expand Up @@ -565,18 +581,22 @@ func TestRoundTripLabelsCacheMiddleware(t *testing.T) {
}{
{name: "first request", req: testRequest, expected: 1},
{name: "same request as the first one, directly use cache", req: testRequest, expected: 1},
{name: "storeMatchers requests won't go to cache", req: testRequestWithStoreMatchers, expected: 2},
{name: "label values request label name foo", req: testLabelValuesRequestFoo, expected: 3},
{name: "same label values query, use cache", req: testLabelValuesRequestFoo, expected: 3},
{name: "label values request different label", req: testLabelValuesRequestBar, expected: 4},
{name: "matchers requests won't go to cache", req: testRequestWithMatchers, expected: 2},
{name: "same matchers requests, use cache", req: testRequestWithMatchers, expected: 2},
{name: "storeMatchers requests won't go to cache", req: testRequestWithStoreMatchers, expected: 3},
{name: "label values request label name foo", req: testLabelValuesRequestFoo, expected: 4},
{name: "same label values query, use cache", req: testLabelValuesRequestFoo, expected: 4},
{name: "label values query with matchers, won't go to cache", req: testLabelValuesRequestFooWithMatchers, expected: 5},
{name: "same label values query with matchers, use cache", req: testLabelValuesRequestFooWithMatchers, expected: 5},
{name: "label values request different label", req: testLabelValuesRequestBar, expected: 6},
{
name: "request but will be partitioned",
req: &ThanosLabelsRequest{
Path: "/api/v1/labels",
Start: 0,
End: 25 * hour,
},
expected: 6,
expected: 8,
},
{
name: "same query as the previous one",
Expand All @@ -585,7 +605,7 @@ func TestRoundTripLabelsCacheMiddleware(t *testing.T) {
Start: 0,
End: 25 * hour,
},
expected: 6,
expected: 8,
},
} {

Expand Down

0 comments on commit 291b930

Please sign in to comment.