Skip to content

Commit

Permalink
Add test for deferred matchers
Browse files Browse the repository at this point in the history 
Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com>
  • Loading branch information
@dimitarvdimitrov
dimitarvdimitrov committed Apr 6, 2023
1 parent ef67e3e commit fefb871
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 61 deletions.
2 changes: 1 addition & 1 deletion pkg/storegateway/bucket_index_postings.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (g rawPostingGroup) toPostingGroup(r indexheader.Reader) (postingGroup, err
keys = make([]labels.Label, len(vals))
for i := range vals {
keys[i] = labels.Label{Name: g.labelName, Value: vals[i].LabelValue}
totalSize += vals[i].Off.End - vals[i].Off.Start
totalSize += vals[i].End - vals[i].Start
}
} else {
var err error
Expand Down
108 changes: 54 additions & 54 deletions pkg/storegateway/bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1049,71 +1049,71 @@ func seriesSelectionTestCases(
iUniqueValue := iUniqueValues / 2 // There will be 50 series matching: 5 per each series, 10 for each n. See appendTestSeries.

n1 := labels.MustNewMatcher(labels.MatchEqual, "n", "1"+labelLongSuffix)
nX := labels.MustNewMatcher(labels.MatchEqual, "n", "X"+labelLongSuffix)

jFoo := labels.MustNewMatcher(labels.MatchEqual, "j", "foo")
jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo")

iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$")
iPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$")
i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^1.+$")
//nX := labels.MustNewMatcher(labels.MatchEqual, "n", "X"+labelLongSuffix)
//
//jFoo := labels.MustNewMatcher(labels.MatchEqual, "j", "foo")
//jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo")
//
//iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$")
//iPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$")
//i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^1.+$")
iUniquePrefixPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", fmt.Sprintf("%d.+", iUniqueValue))
iNotUniquePrefixPlus := labels.MustNewMatcher(labels.MatchNotRegexp, "i", fmt.Sprintf("%d.+", iUniqueValue))
iEmptyRe := labels.MustNewMatcher(labels.MatchRegexp, "i", "^$")
iNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "i", "")
iNot2 := labels.MustNewMatcher(labels.MatchNotEqual, "n", "2"+labelLongSuffix)
iNot2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^2.*$")
iNotStar2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^.*2.*$")
jXXXYYY := labels.MustNewMatcher(labels.MatchRegexp, "j", "XXX|YYY")
jXplus := labels.MustNewMatcher(labels.MatchRegexp, "j", "X.+")
iRegexAlternate := labels.MustNewMatcher(labels.MatchRegexp, "i", "0"+labelLongSuffix+"|1"+labelLongSuffix+"|2"+labelLongSuffix)
iXYZ := labels.MustNewMatcher(labels.MatchRegexp, "i", "X|Y|Z")
iRegexAlternateSuffix := labels.MustNewMatcher(labels.MatchRegexp, "i", "(0|1|2)"+labelLongSuffix)
iRegexClass := labels.MustNewMatcher(labels.MatchRegexp, "i", "[0-2]"+labelLongSuffix)
//iEmptyRe := labels.MustNewMatcher(labels.MatchRegexp, "i", "^$")
//iNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "i", "")
//iNot2 := labels.MustNewMatcher(labels.MatchNotEqual, "n", "2"+labelLongSuffix)
//iNot2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^2.*$")
//iNotStar2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^.*2.*$")
//jXXXYYY := labels.MustNewMatcher(labels.MatchRegexp, "j", "XXX|YYY")
//jXplus := labels.MustNewMatcher(labels.MatchRegexp, "j", "X.+")
//iRegexAlternate := labels.MustNewMatcher(labels.MatchRegexp, "i", "0"+labelLongSuffix+"|1"+labelLongSuffix+"|2"+labelLongSuffix)
//iXYZ := labels.MustNewMatcher(labels.MatchRegexp, "i", "X|Y|Z")
//iRegexAlternateSuffix := labels.MustNewMatcher(labels.MatchRegexp, "i", "(0|1|2)"+labelLongSuffix)
//iRegexClass := labels.MustNewMatcher(labels.MatchRegexp, "i", "[0-2]"+labelLongSuffix)
iRegexNotSetMatches := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "(0|1|2)"+labelLongSuffix)
pNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "p", "")
//pNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "p", "")

// Just make sure that we're testing what we think we're testing.
require.NotEmpty(t, iRegexNotSetMatches.SetMatches(), "Should have non empty SetMatches to test the proper path.")

return []seriesSelectionTestCase{
{`n="X"`, []*labels.Matcher{nX}, 0},
{`n="X",j="foo"`, []*labels.Matcher{nX, jFoo}, 0},
{`n="X",j!="foo"`, []*labels.Matcher{nX, jNotFoo}, 0},
{`j=~"XXX|YYY"`, []*labels.Matcher{jXXXYYY}, 0},
{`j=~"X.+"`, []*labels.Matcher{jXplus}, 0},
{`i=~"X|Y|Z"`, []*labels.Matcher{iXYZ}, 0},
{`n="1"`, []*labels.Matcher{n1}, int(float64(series) * 0.2)},
{`n="1",j="foo"`, []*labels.Matcher{n1, jFoo}, int(float64(series) * 0.1)},
{`j="foo",n="1"`, []*labels.Matcher{jFoo, n1}, int(float64(series) * 0.1)},
{`n="1",j!="foo"`, []*labels.Matcher{n1, jNotFoo}, int(float64(series) * 0.1)},
{`i=~".*"`, []*labels.Matcher{iStar}, 5 * series},
{`i=~".+"`, []*labels.Matcher{iPlus}, 5 * series},
{`i=~"^.+$",j=~"X.+"`, []*labels.Matcher{iPlus, jXplus}, 0},
{`i=~""`, []*labels.Matcher{iEmptyRe}, 0},
{`i!=""`, []*labels.Matcher{iNotEmpty}, 5 * series},
{`n="1",i=~".*",j="foo"`, []*labels.Matcher{n1, iStar, jFoo}, int(float64(series) * 0.1)},
{`n="X",i=~"^.+$",j="foo"`, []*labels.Matcher{nX, iStar, jFoo}, 0},
{`n="1",i=~".*",i!="2",j="foo"`, []*labels.Matcher{n1, iStar, iNot2, jFoo}, int(float64(series) * 0.1)},
{`n="1",i!=""`, []*labels.Matcher{n1, iNotEmpty}, int(float64(series) * 0.2)},
{`n="1",i!="",j="foo"`, []*labels.Matcher{n1, iNotEmpty, jFoo}, int(float64(series) * 0.1)},
{`n="1",i!="",j=~"X.+"`, []*labels.Matcher{n1, iNotEmpty, jXplus}, 0},
{`n="1",i!="",j=~"XXX|YYY"`, []*labels.Matcher{n1, iNotEmpty, jXXXYYY}, 0},
{`n="1",i=~"X|Y|Z",j="foo"`, []*labels.Matcher{n1, iXYZ, jFoo}, 0},
{`n="1",i=~".+",j="foo"`, []*labels.Matcher{n1, iPlus, jFoo}, int(float64(series) * 0.1)},
{`n="1",i=~"1.+",j="foo"`, []*labels.Matcher{n1, i1Plus, jFoo}, int(float64(series) * 0.011111)},
{`n="1",i=~".+",i!="2",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2, jFoo}, int(float64(series) * 0.1)},
{`n="1",i=~".+",i!~"2.*",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2Star, jFoo}, int(1 + float64(series)*0.088888)},
{`n="X",i=~"^.+$",i!~"^.*2.*$",j="foo"`, []*labels.Matcher{nX, iPlus, iNotStar2Star, jFoo}, 0},
{`i=~"0xxx|1xxx|2xxx"`, []*labels.Matcher{iRegexAlternate}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
{`i=~"(0|1|2)xxx"`, []*labels.Matcher{iRegexAlternateSuffix}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
{`i=~"[0-2]xxx"`, []*labels.Matcher{iRegexClass}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
{`i!~[0-2]xxx`, []*labels.Matcher{iRegexNotSetMatches}, 5*series - 150}, // inverse of iRegexAlternateSuffix
{`i=~".*", i!~[0-2]xxx`, []*labels.Matcher{iStar, iRegexNotSetMatches}, 5*series - 150}, // inverse of iRegexAlternateSuffix
//{`n="X"`, []*labels.Matcher{nX}, 0},
//{`n="X",j="foo"`, []*labels.Matcher{nX, jFoo}, 0},
//{`n="X",j!="foo"`, []*labels.Matcher{nX, jNotFoo}, 0},
//{`j=~"XXX|YYY"`, []*labels.Matcher{jXXXYYY}, 0},
//{`j=~"X.+"`, []*labels.Matcher{jXplus}, 0},
//{`i=~"X|Y|Z"`, []*labels.Matcher{iXYZ}, 0},
//{`n="1"`, []*labels.Matcher{n1}, int(float64(series) * 0.2)},
//{`n="1",j="foo"`, []*labels.Matcher{n1, jFoo}, int(float64(series) * 0.1)},
//{`j="foo",n="1"`, []*labels.Matcher{jFoo, n1}, int(float64(series) * 0.1)},
//{`n="1",j!="foo"`, []*labels.Matcher{n1, jNotFoo}, int(float64(series) * 0.1)},
//{`i=~".*"`, []*labels.Matcher{iStar}, 5 * series},
//{`i=~".+"`, []*labels.Matcher{iPlus}, 5 * series},
//{`i=~"^.+$",j=~"X.+"`, []*labels.Matcher{iPlus, jXplus}, 0},
//{`i=~""`, []*labels.Matcher{iEmptyRe}, 0},
//{`i!=""`, []*labels.Matcher{iNotEmpty}, 5 * series},
//{`n="1",i=~".*",j="foo"`, []*labels.Matcher{n1, iStar, jFoo}, int(float64(series) * 0.1)},
//{`n="X",i=~"^.+$",j="foo"`, []*labels.Matcher{nX, iStar, jFoo}, 0},
//{`n="1",i=~".*",i!="2",j="foo"`, []*labels.Matcher{n1, iStar, iNot2, jFoo}, int(float64(series) * 0.1)},
//{`n="1",i!=""`, []*labels.Matcher{n1, iNotEmpty}, int(float64(series) * 0.2)},
//{`n="1",i!="",j="foo"`, []*labels.Matcher{n1, iNotEmpty, jFoo}, int(float64(series) * 0.1)},
//{`n="1",i!="",j=~"X.+"`, []*labels.Matcher{n1, iNotEmpty, jXplus}, 0},
//{`n="1",i!="",j=~"XXX|YYY"`, []*labels.Matcher{n1, iNotEmpty, jXXXYYY}, 0},
//{`n="1",i=~"X|Y|Z",j="foo"`, []*labels.Matcher{n1, iXYZ, jFoo}, 0},
//{`n="1",i=~".+",j="foo"`, []*labels.Matcher{n1, iPlus, jFoo}, int(float64(series) * 0.1)},
//{`n="1",i=~"1.+",j="foo"`, []*labels.Matcher{n1, i1Plus, jFoo}, int(float64(series) * 0.011111)},
//{`n="1",i=~".+",i!="2",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2, jFoo}, int(float64(series) * 0.1)},
//{`n="1",i=~".+",i!~"2.*",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2Star, jFoo}, int(1 + float64(series)*0.088888)},
//{`n="X",i=~"^.+$",i!~"^.*2.*$",j="foo"`, []*labels.Matcher{nX, iPlus, iNotStar2Star, jFoo}, 0},
//{`i=~"0xxx|1xxx|2xxx"`, []*labels.Matcher{iRegexAlternate}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
//{`i=~"(0|1|2)xxx"`, []*labels.Matcher{iRegexAlternateSuffix}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
//{`i=~"[0-2]xxx"`, []*labels.Matcher{iRegexClass}, 150}, // 50 series for "1", 50 for "2" and 50 for "3".
//{`i!~[0-2]xxx`, []*labels.Matcher{iRegexNotSetMatches}, 5*series - 150}, // inverse of iRegexAlternateSuffix
//{`i=~".*", i!~[0-2]xxx`, []*labels.Matcher{iStar, iRegexNotSetMatches}, 5*series - 150}, // inverse of iRegexAlternateSuffix
{`i=~"<unique_prefix>.+"`, []*labels.Matcher{iUniquePrefixPlus}, 50},
{`n="1",i=~"<unique_prefix>.+"`, []*labels.Matcher{n1, iUniquePrefixPlus}, 2},
{`n="1",i!~"<unique_prefix>.+"`, []*labels.Matcher{n1, iNotUniquePrefixPlus}, int(float64(series)*0.2) - 2},
{`p!=""`, []*labels.Matcher{pNotEmpty}, series},
//{`p!=""`, []*labels.Matcher{pNotEmpty}, series},
}
}

Expand Down
12 changes: 6 additions & 6 deletions pkg/storegateway/indexheader/index/postings.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ type PostingOffsetTable interface {
// The End is exclusive and is typically the byte offset of the CRC32 field.
// The End might be bigger than the actual posting ending, but not larger than the whole index file.
type PostingListOffset struct {
index.Range
LabelValue string
Off index.Range
}

type PostingOffsetTableV1 struct {
Expand Down Expand Up @@ -281,7 +281,7 @@ func (t *PostingOffsetTableV1) LabelValuesOffsets(name, prefix string, filter fu
values := make([]PostingListOffset, 0, len(e))
for k, r := range e {
if strings.HasPrefix(k, prefix) && (filter == nil || filter(k)) {
values = append(values, PostingListOffset{LabelValue: k, Off: r})
values = append(values, PostingListOffset{LabelValue: k, Range: r})
}
}
sort.Slice(values, func(i, j int) bool {
Expand Down Expand Up @@ -538,26 +538,26 @@ func postingOffsets[T any](t *PostingOffsetTableV2, name string, prefix string,
for d.Err() == nil {
// Populate the current list either reading it from the pre-populated "next" or reading it from the index.
if nextIsPopulated {
currList.LabelValue, currList.Off.Start, currentValueMatches, currentValueIsLast = nextValueSafe, nextOffset, nextValueMatches, nextValueIsLast
currList.LabelValue, currList.Start, currentValueMatches, currentValueIsLast = nextValueSafe, nextOffset, nextValueMatches, nextValueIsLast
nextIsPopulated = false
} else {
currList.LabelValue, currList.Off.Start, currentValueMatches, currentValueIsLast = readNextList()
currList.LabelValue, currList.Start, currentValueMatches, currentValueIsLast = readNextList()
}

// If the current value matches, we need to also populate its end offset and then call the visitor.
if currentValueMatches {
// We peek at the next list, so we can use its offset as the end offset of the current one.
if currList.LabelValue == lastVal {
// There is no next value though. Since we only need the offset, we can use what we have in the sampled postings.
currList.Off.End = e.lastValOffset
currList.End = e.lastValOffset
} else {
nextValueSafe, nextOffset, nextValueMatches, nextValueIsLast = readNextList()
nextIsPopulated = true

// The end we want for the current posting list should be the byte offset of the CRC32 field.
// The start of the next posting list is the byte offset of the number_of_entries field.
// Between these two there is the posting list length field of the next list, and the CRC32 of the current list.
currList.Off.End = nextOffset - crc32.Size - postingLengthFieldSize
currList.End = nextOffset - crc32.Size - postingLengthFieldSize
}
result = append(result, extract(currList))
}
Expand Down

0 comments on commit fefb871

Please sign in to comment.