streaming store-gateway: avoid copying postings when hashing them

pkg/storegateway/indexcache/cache.go

@@ -7,10 +7,11 @@ package indexcache
 
 import (
 	"context"
-	"crypto/sha1"
 	"encoding/base64"
+	"reflect"
 	"sort"
 	"strings"
+	"unsafe"
 
 	"github.com/oklog/ulid"
 	"github.com/prometheus/client_golang/prometheus"
@@ -91,23 +92,20 @@ type PostingsKey string
 
 // CanonicalPostingsKey creates a canonical version of PostingsKey
 func CanonicalPostingsKey(postings []storage.SeriesRef) PostingsKey {
-	hashable := make([]byte, len(postings)*8)
-	for i, posting := range postings {
-		for octet := 0; octet < 8; octet++ {
-			hashable[i+octet] = byte(posting >> (octet * 8))
-		}
-	}
-
-	// We hash the postings list twice to minimize the chance of collisions
-	hasher1, _ := blake2b.New256(nil) // This will never return an error
-	hasher2 := sha1.New()
-
-	_, _ = hasher1.Write(hashable)
-	_, _ = hasher2.Write(hashable)
-
-	checksum := hasher2.Sum(hasher1.Sum(nil))
+	hashable := unsafeCastPostingsToBytes(postings)
+	checksum := blake2b.Sum256(hashable)
+	return PostingsKey(base64.RawURLEncoding.EncodeToString(checksum[:]))
+}
 
-	return PostingsKey(base64.RawURLEncoding.EncodeToString(checksum))
+// unsafeCastPostingsToBytes returns the postings as a slice of bytes with minimal allocations.
+// It casts the memory region of the underlying array to a slice of bytes.
+func unsafeCastPostingsToBytes(postings []storage.SeriesRef) []byte {
+	byteSlice := make([]byte, 0)
+	slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(&byteSlice))
+	slicePtr.Data = (*reflect.SliceHeader)(unsafe.Pointer(&postings)).Data
+	slicePtr.Len = len(postings) * 8
+	slicePtr.Cap = slicePtr.Len
+	return byteSlice
 }
 
 // LabelMatchersKey represents a canonical key for a []*matchers.Matchers slice

pkg/storegateway/indexcache/cache_test.go

@@ -55,3 +55,34 @@ func BenchmarkCanonicalPostingsKey(b *testing.B) {
 		})
 	}
 }
+
+func TestUnsafeCastPostingsToBytes(t *testing.T) {
+	slowPostingsToBytes := func(postings []storage.SeriesRef) []byte {
+		byteSlice := make([]byte, len(postings)*8)
+		for i, posting := range postings {
+			for octet := 0; octet < 8; octet++ {
+				byteSlice[i*8+octet] = byte(posting >> (octet * 8))
+			}
+		}
+		return byteSlice
+	}
+	t.Run("base case", func(t *testing.T) {
+		postings := []storage.SeriesRef{1, 2}
+		assert.Equal(t, slowPostingsToBytes(postings), unsafeCastPostingsToBytes(postings))
+	})
+	t.Run("zero-length postings", func(t *testing.T) {
+		postings := make([]storage.SeriesRef, 0)
+		assert.Equal(t, slowPostingsToBytes(postings), unsafeCastPostingsToBytes(postings))
+	})
+	t.Run("nil postings", func(t *testing.T) {
+		assert.Equal(t, []byte(nil), unsafeCastPostingsToBytes(nil))
+	})
+	t.Run("more than 256 postings", func(t *testing.T) {
+		// Only casting a slice pointer truncates all postings to only their last byte.
+		postings := make([]storage.SeriesRef, 300)
+		for i := range postings {
+			postings[i] = storage.SeriesRef(i + 1)
+		}
+		assert.Equal(t, slowPostingsToBytes(postings), unsafeCastPostingsToBytes(postings))
+	})
+}