Switch to 24 byte node IDs

This will allow us to accomodate leaf nodes bigger than 4GB, which is
conceivable if a tree is really poorly dimensioned. It will increase the
size of inner nodes by an additional 512 bytes but that will probably be
swamped by the volume of stats.

This change will break any existing data directories.

nodestore/nodeid.go

@@ -9,40 +9,40 @@ import (
 )
 
 /*
-Node IDs in dp3 are 16 bytes long. The first 8 bytes is an unsigned 64-bit
-integer interpreted as an object identifier in storage. The next 4 bytes is an
-offset, and the final 4 bytes is a length to read.
+Node IDs in dp3 are 24 bytes long. The first 8 bytes is an unsigned 64-bit
+integer interpreted as an object identifier in storage. The next 8 bytes is an
+offset, and the final 8 bytes is a length to read.
 
 With this structure it is possible to resolve any node ID directly to a storage
 location. Many nodes may be stored in a single object.
 */
 
 ////////////////////////////////////////////////////////////////////////////////
 
-// NodeID is a 16-byte identifier for a node in the nodestore.
-type NodeID [16]byte
+// NodeID is a 24-byte identifier for a node in the nodestore.
+type NodeID [24]byte
 
 // OID returns the object identifier of the node.
 func (n NodeID) OID() string {
 	return strconv.FormatUint(binary.LittleEndian.Uint64(n[:8]), 10)
 }
 
 // Offset returns the offset of the node.
-func (n NodeID) Offset() int {
-	return int(binary.LittleEndian.Uint32(n[8:]))
+func (n NodeID) Offset() uint64 {
+	return binary.LittleEndian.Uint64(n[8:])
 }
 
 // Length returns the length of the node.
-func (n NodeID) Length() int {
-	return int(binary.LittleEndian.Uint32(n[12:]))
+func (n NodeID) Length() uint64 {
+	return binary.LittleEndian.Uint64(n[16:])
 }
 
 // NewNodeID creates a new node ID from an object ID, offset, and length.
-func NewNodeID(oid uint64, offset, length int) NodeID {
+func NewNodeID(oid, offset, length uint64) NodeID {
 	var id NodeID
 	binary.LittleEndian.PutUint64(id[:8], oid)
-	binary.LittleEndian.PutUint32(id[8:12], uint32(offset))
-	binary.LittleEndian.PutUint32(id[12:], uint32(length))
+	binary.LittleEndian.PutUint64(id[8:], offset)
+	binary.LittleEndian.PutUint64(id[16:], length)
 	return id
 }
 
@@ -60,9 +60,9 @@ func (n *NodeID) UnmarshalJSON(data []byte) error {
 	s = s[1 : len(s)-1]
 	parts := strings.Split(s, ":")
 	oid, _ := strconv.ParseUint(parts[0], 10, 64)
-	offset, _ := strconv.ParseUint(parts[1], 10, 32)
-	length, _ := strconv.ParseUint(parts[2], 10, 32)
-	nodeID := generateNodeID(objectID(oid), int(offset), int(length))
+	offset, _ := strconv.ParseUint(parts[1], 10, 64)
+	length, _ := strconv.ParseUint(parts[2], 10, 64)
+	nodeID := generateNodeID(objectID(oid), offset, length)
 	*n = nodeID
 	return nil
 }
@@ -76,9 +76,9 @@ func (n *NodeID) Scan(value interface{}) error {
 	}
 	parts := strings.Split(s, ":")
 	oid, _ := strconv.ParseUint(parts[0], 10, 64)
-	offset, _ := strconv.ParseUint(parts[1], 10, 32)
-	length, _ := strconv.ParseUint(parts[2], 10, 32)
-	nodeID := generateNodeID(objectID(oid), int(offset), int(length))
+	offset, _ := strconv.ParseUint(parts[1], 10, 64)
+	length, _ := strconv.ParseUint(parts[2], 10, 64)
+	nodeID := generateNodeID(objectID(oid), offset, length)
 	*n = nodeID
 	return nil
 }
@@ -88,10 +88,10 @@ func (n NodeID) Value() (driver.Value, error) {
 	return driver.Value(n.String()), nil
 }
 
-func generateNodeID(oid objectID, offset int, length int) NodeID {
+func generateNodeID(oid objectID, offset uint64, length uint64) NodeID {
 	var id NodeID
 	binary.LittleEndian.PutUint64(id[:], uint64(oid))
-	binary.LittleEndian.PutUint32(id[8:], uint32(offset))
-	binary.LittleEndian.PutUint32(id[12:], uint32(length))
+	binary.LittleEndian.PutUint64(id[8:], offset)
+	binary.LittleEndian.PutUint64(id[16:], length)
 	return id
 }

nodestore/nodestore.go

@@ -79,7 +79,7 @@ func (n *Nodestore) Get(ctx context.Context, id NodeID) (Node, error) {
 	if value, ok := n.cache.Get(id); ok {
 		return value, nil
 	}
-	reader, err := n.store.GetRange(ctx, id.OID(), id.Offset(), id.Length())
+	reader, err := n.store.GetRange(ctx, id.OID(), int(id.Offset()), int(id.Length()))
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotFound) {
 			return nil, NodeNotFoundError{id}
@@ -104,7 +104,7 @@ func (n *Nodestore) Get(ctx context.Context, id NodeID) (Node, error) {
 // does not cache data.
 func (n *Nodestore) GetLeafData(ctx context.Context, id NodeID) (io.ReadSeekCloser, error) {
 	// NB: Add one to the offset to skip the type byte.
-	reader, err := n.store.GetRange(ctx, id.OID(), id.Offset()+1, id.Length()-1)
+	reader, err := n.store.GetRange(ctx, id.OID(), int(id.Offset())+1, int(id.Length())-1)
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotFound) {
 			return nil, NodeNotFoundError{id}

nodestore/nodestore_test.go

@@ -38,7 +38,7 @@ func TestNodestore(t *testing.T) {
 		bytes := node.ToBytes()
 		_, err = buf.Write(bytes)
 		require.NoError(t, err)
-		addr := nodestore.NewNodeID(1, n, len(bytes))
+		addr := nodestore.NewNodeID(1, uint64(n), uint64(len(bytes)))
 
 		require.NoError(t, ns.Put(ctx, 1, buf.Bytes()))
 

nodestore/testutils.go

@@ -18,7 +18,7 @@ func MockNodestore(ctx context.Context, t *testing.T) *Nodestore {
 }
 
 func RandomNodeID() NodeID {
-	buf := [16]byte{}
+	buf := [24]byte{}
 	_, _ = rand.Read(buf[:])
 	return NodeID(buf)
 }

tree/bytetree.go

@@ -70,8 +70,8 @@ func (b *byteTree) Put(ctx context.Context, id nodestore.NodeID, node nodestore.
 }
 
 func (b *byteTree) Root() nodestore.NodeID {
-	_, _ = b.r.Seek(-16, io.SeekEnd)
-	id := make([]byte, 16)
+	_, _ = b.r.Seek(-24, io.SeekEnd)
+	id := make([]byte, 24)
 	_, _ = io.ReadFull(b.r, id)
 	return nodestore.NodeID(id)
 }

tree/memtree.go

@@ -94,7 +94,7 @@ func (m *MemTree) FromBytes(ctx context.Context, data []byte) error {
 
 // ToBytes serializes the memtree to a byte array suitable for storage in the
 // WAL or in an object. The nodes are serialized from leaf to root. The IDs are
-// boosted by offset. The last 16 bytes are the root ID.
+// boosted by offset. The last 24 bytes are the root ID.
 func (m *MemTree) ToBytes(ctx context.Context, oid uint64) ([]byte, error) { // nolint: funlen
 	root, err := m.Get(ctx, m.root)
 	if err != nil {
@@ -147,7 +147,7 @@ func (m *MemTree) ToBytes(ctx context.Context, oid uint64) ([]byte, error) { //
 		if err != nil {
 			return nil, fmt.Errorf("failed to write node: %w", err)
 		}
-		nodeID := nodestore.NewNodeID(oid, offset, n)
+		nodeID := nodestore.NewNodeID(oid, uint64(offset), uint64(n))
 		offset += n
 		processed[id] = nodeID
 	}

treemgr/treemgr.go

@@ -302,7 +302,7 @@ func (tm *TreeManager) newRoot(
 	if err := tm.ns.Put(ctx, version, data); err != nil {
 		return nodestore.NodeID{}, 0, fmt.Errorf("failed to put root: %w", err)
 	}
-	nodeID := nodestore.NewNodeID(version, 0, len(data))
+	nodeID := nodestore.NewNodeID(version, 0, uint64(len(data)))
 	return nodeID, version, nil
 }
 
@@ -342,8 +342,8 @@ func (tm *TreeManager) mergeBatch(ctx context.Context, batch *wal.Batch) error {
 	if err != nil {
 		return fmt.Errorf("failed to serialize partial tree: %w", err)
 	}
-	rootID := data[len(data)-16:]
-	if err := tm.ns.Put(ctx, version, data[:len(data)-16]); err != nil {
+	rootID := data[len(data)-24:]
+	if err := tm.ns.Put(ctx, version, data[:len(data)-24]); err != nil {
 		return fmt.Errorf("failed to put tree: %w", err)
 	}
 	if err := tm.rootmap.Put(ctx, batch.ProducerID, batch.Topic, version, nodestore.NodeID(rootID)); err != nil {