pbss: serialized points in uncompressed form (#370)

* config: rename serialized point size constant and value to uncompressed form

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* switch to uncompressed serialization

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

---------

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

encoding.go

@@ -44,16 +44,16 @@ const (
 	nodeTypeOffset = 0
 
 	// Internal nodes offsets.
-	internalBitlistOffset      = nodeTypeOffset + nodeTypeSize
-	internalNodeChildrenOffset = internalBitlistOffset + bitlistSize
+	internalBitlistOffset    = nodeTypeOffset + nodeTypeSize
+	internalCommitmentOffset = internalBitlistOffset + bitlistSize
 
 	// Leaf node offsets.
 	leafSteamOffset        = nodeTypeOffset + nodeTypeSize
 	leafBitlistOffset      = leafSteamOffset + StemSize
 	leafCommitmentOffset   = leafBitlistOffset + bitlistSize
-	leafC1CommitmentOffset = leafCommitmentOffset + SerializedPointCompressedSize
-	leafC2CommitmentOffset = leafC1CommitmentOffset + SerializedPointCompressedSize
-	leafChildrenOffset     = leafC2CommitmentOffset + SerializedPointCompressedSize
+	leafC1CommitmentOffset = leafCommitmentOffset + SerializedPointUncompressedSize
+	leafC2CommitmentOffset = leafC1CommitmentOffset + SerializedPointUncompressedSize
+	leafChildrenOffset     = leafC2CommitmentOffset + SerializedPointUncompressedSize
 )
 
 func bit(bitlist []byte, nr int) bool {
@@ -67,19 +67,19 @@ var errSerializedPayloadTooShort = errors.New("verkle payload is too short")
 
 // ParseNode deserializes a node into its proper VerkleNode instance.
 // The serialized bytes have the format:
-// - Internal nodes: <nodeType><bitlist><children...>
-// - Leaf nodes:     <nodeType><stem><bitlist><c1comm><c2comm><children...>
+// - Internal nodes: <nodeType><bitlist><commitment>
+// - Leaf nodes:     <nodeType><stem><bitlist><comm><c1comm><c2comm><children...>
 func ParseNode(serializedNode []byte, depth byte) (VerkleNode, error) {
 	// Check that the length of the serialized node is at least the smallest possible serialized node.
-	if len(serializedNode) < nodeTypeSize+SerializedPointCompressedSize {
+	if len(serializedNode) < nodeTypeSize+SerializedPointUncompressedSize {
 		return nil, errSerializedPayloadTooShort
 	}
 
 	switch serializedNode[0] {
 	case leafRLPType:
 		return parseLeafNode(serializedNode, depth)
 	case internalRLPType:
-		return CreateInternalNode(serializedNode[internalBitlistOffset:internalNodeChildrenOffset], serializedNode[internalNodeChildrenOffset:], depth)
+		return CreateInternalNode(serializedNode[internalBitlistOffset:internalCommitmentOffset], serializedNode[internalCommitmentOffset:], depth)
 	default:
 		return nil, ErrInvalidNodeEncoding
 	}
@@ -101,11 +101,17 @@ func parseLeafNode(serialized []byte, depth byte) (VerkleNode, error) {
 	ln := NewLeafNodeWithNoComms(serialized[leafSteamOffset:leafSteamOffset+StemSize], values[:])
 	ln.setDepth(depth)
 	ln.c1 = new(Point)
-	ln.c1.SetBytesTrusted(serialized[leafC1CommitmentOffset : leafC1CommitmentOffset+SerializedPointCompressedSize])
+
+	// Sanity check that we have at least 3*SerializedPointUncompressedSize bytes left in the serialized payload.
+	if len(serialized[leafCommitmentOffset:]) < 3*SerializedPointUncompressedSize {
+		return nil, fmt.Errorf("leaf node commitments are not the correct size, expected at least %d, got %d", 3*SerializedPointUncompressedSize, len(serialized[leafC1CommitmentOffset:]))
+	}
+
+	ln.c1.SetBytesUncompressed(serialized[leafC1CommitmentOffset:leafC1CommitmentOffset+SerializedPointUncompressedSize], true)
 	ln.c2 = new(Point)
-	ln.c2.SetBytesTrusted(serialized[leafC2CommitmentOffset : leafC2CommitmentOffset+SerializedPointCompressedSize])
+	ln.c2.SetBytesUncompressed(serialized[leafC2CommitmentOffset:leafC2CommitmentOffset+SerializedPointUncompressedSize], true)
 	ln.commitment = new(Point)
-	ln.commitment.SetBytesTrusted(serialized[leafCommitmentOffset:leafC1CommitmentOffset])
+	ln.commitment.SetBytesUncompressed(serialized[leafCommitmentOffset:leafC1CommitmentOffset], true)
 	return ln, nil
 }
 
@@ -132,11 +138,11 @@ func CreateInternalNode(bitlist []byte, raw []byte, depth byte) (*InternalNode,
 		}
 	}
 	node.depth = depth
-	if len(raw) != SerializedPointCompressedSize {
+	if len(raw) != SerializedPointUncompressedSize {
 		return nil, ErrInvalidNodeEncoding
 	}
 
 	node.commitment = new(Point)
-	node.commitment.SetBytesTrusted(raw)
+	node.commitment.SetBytesUncompressed(raw, true)
 	return node, nil
 }

encoding_test.go

@@ -18,8 +18,8 @@ func TestLeafStemLength(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(ser) != nodeTypeSize+StemSize+bitlistSize+3*SerializedPointCompressedSize {
-		t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), nodeTypeSize+StemSize+bitlistSize+2*SerializedPointCompressedSize)
+	if len(ser) != nodeTypeSize+StemSize+bitlistSize+3*SerializedPointUncompressedSize {
+		t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), nodeTypeSize+StemSize+bitlistSize+2*SerializedPointUncompressedSize)
 	}
 }
 

go.mod

@@ -2,7 +2,7 @@ module github.com/gballet/go-verkle
 
 go 1.18
 
-require github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80
+require github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df
 
 require (
 	golang.org/x/sync v0.1.0 // indirect

go.sum

@@ -1,5 +1,5 @@
-github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0=
-github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI=
+github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df h1:MV8zGEpmkuAocFqFclIJ3zUz1+6rv0T5QYWD9UDDRVE=
+github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=

hashednode.go

@@ -53,7 +53,7 @@ func (n *HashedNode) Commit() *Point {
 	}
 	if n.cachedPoint == nil {
 		n.cachedPoint = new(Point)
-		n.cachedPoint.SetBytesTrusted(n.commitment)
+		n.cachedPoint.SetBytesUncompressed(n.commitment, true)
 	}
 	return n.cachedPoint
 }

ipa.go

@@ -38,7 +38,7 @@ type (
 )
 
 const (
-	SerializedPointCompressedSize = 32
+	SerializedPointUncompressedSize = 64
 )
 
 func CopyFr(dst, src *Fr) {

tree.go

@@ -512,7 +512,7 @@ func (n *InternalNode) toHashedNode() *HashedNode {
 	if n.commitment == nil {
 		panic("nil commitment")
 	}
-	comm := n.commitment.Bytes()
+	comm := n.commitment.BytesUncompressed()
 	return &HashedNode{commitment: comm[:]}
 }
 
@@ -843,21 +843,25 @@ func (n *InternalNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]
 	return pe, esses, poass, nil
 }
 
+// Serialize returns the serialized form of the internal node.
+// The format is: <nodeType><bitlist><commitment>
 func (n *InternalNode) Serialize() ([]byte, error) {
-	ret := make([]byte, nodeTypeSize+bitlistSize+SerializedPointCompressedSize)
-	bitlist := ret[internalBitlistOffset:internalNodeChildrenOffset]
+	ret := make([]byte, nodeTypeSize+bitlistSize+SerializedPointUncompressedSize)
+
+	// Write the <bitlist>.
+	bitlist := ret[internalBitlistOffset:internalCommitmentOffset]
 	for i, c := range n.children {
 		if _, ok := c.(Empty); !ok {
 			setBit(bitlist, i)
 		}
 	}
 
-	// Store in ret the serialized result
+	// Write the <node-type>
 	ret[nodeTypeOffset] = internalRLPType
-	comm := n.commitment.Bytes()
-	// XXX rename
-	copy(ret[internalNodeChildrenOffset:], comm[:])
-	// Note that children were already appended in ret through the children slice.
+
+	// Write the <commitment>
+	comm := n.commitment.BytesUncompressed()
+	copy(ret[internalCommitmentOffset:], comm[:])
 
 	return ret, nil
 }
@@ -941,7 +945,7 @@ func (n *LeafNode) ToHashedNode() *HashedNode {
 	if n.commitment == nil {
 		panic("nil commitment")
 	}
-	comm := n.commitment.Bytes()
+	comm := n.commitment.BytesUncompressed()
 	return &HashedNode{commitment: comm[:]}
 }
 
@@ -1386,10 +1390,10 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte
 }
 
 // Serialize serializes a LeafNode.
-// The format is: <nodeType><stem><bitlist><c1comm><c2comm><children...>
+// The format is: <nodeType><stem><bitlist><comm><c1comm><c2comm><children...>
 func (n *LeafNode) Serialize() ([]byte, error) {
-	cBytes := banderwagon.ElementsToBytes([]*banderwagon.Element{n.commitment, n.c1, n.c2})
-	return n.serializeLeafWithCompressedCommitments(cBytes[0], cBytes[1], cBytes[2]), nil
+	cBytes := banderwagon.ElementsToBytesUncompressed([]*banderwagon.Element{n.commitment, n.c1, n.c2})
+	return n.serializeLeafWithUncompressedCommitments(cBytes[0], cBytes[1], cBytes[2]), nil
 }
 
 func (n *LeafNode) Copy() VerkleNode {
@@ -1467,7 +1471,7 @@ func ToDot(root VerkleNode) string {
 // Providing both allows this library to do more optimizations.
 type SerializedNode struct {
 	Node            VerkleNode
-	CommitmentBytes [32]byte
+	CommitmentBytes [SerializedPointUncompressedSize]byte
 	Path            []byte
 	SerializedBytes []byte
 }
@@ -1485,21 +1489,20 @@ func (n *InternalNode) BatchSerialize() ([]SerializedNode, error) {
 
 	// We collect all the *Point, so we can batch all projective->affine transformations.
 	pointsToCompress := make([]*Point, 0, 3*len(nodes))
-	// Contains a map between VerkleNode and the index in the `compressedPoints` containing the commitment below.
-	compressedPointsIdxs := make(map[VerkleNode]int, 3*len(nodes))
+	// Contains a map between VerkleNode and the index in the serializedPoints containing the commitment below.
+	serializedPointsIdxs := make(map[VerkleNode]int, len(nodes))
 	for i := range nodes {
 		switch n := nodes[i].(type) {
 		case *InternalNode:
 			pointsToCompress = append(pointsToCompress, n.commitment)
-			compressedPointsIdxs[n] = len(pointsToCompress) - 1
+			serializedPointsIdxs[n] = len(pointsToCompress) - 1
 		case *LeafNode:
 			pointsToCompress = append(pointsToCompress, n.commitment, n.c1, n.c2)
-			compressedPointsIdxs[n] = len(pointsToCompress) - 3
 		}
 	}
 
 	// Now we do the all transformations in a single-shot.
-	compressedPoints := banderwagon.ElementsToBytes(pointsToCompress)
+	serializedPoints := banderwagon.ElementsToBytesUncompressed(pointsToCompress)
 
 	// Now we that we did the heavy CPU work, we have to do the rest of `nodes` serialization
 	// taking the compressed points from this single list.
@@ -1508,27 +1511,27 @@ func (n *InternalNode) BatchSerialize() ([]SerializedNode, error) {
 	for i := range nodes {
 		switch n := nodes[i].(type) {
 		case *InternalNode:
-			serialized, err := n.serializeInternalWithCompressedCommitment(compressedPointsIdxs, compressedPoints)
+			serialized, err := n.serializeInternalWithUncompressedCommitment(serializedPointsIdxs, serializedPoints)
 			if err != nil {
 				return nil, err
 			}
 			sn := SerializedNode{
 				Node:            n,
 				Path:            paths[i],
-				CommitmentBytes: compressedPoints[idx],
+				CommitmentBytes: serializedPoints[idx],
 				SerializedBytes: serialized,
 			}
 			ret = append(ret, sn)
 			idx++
 		case *LeafNode:
-			cBytes := compressedPoints[idx]
-			c1Bytes := compressedPoints[idx+1]
-			c2Bytes := compressedPoints[idx+2]
+			cBytes := serializedPoints[idx]
+			c1Bytes := serializedPoints[idx+1]
+			c2Bytes := serializedPoints[idx+2]
 			sn := SerializedNode{
 				Node:            n,
 				Path:            paths[i],
-				CommitmentBytes: compressedPoints[idx],
-				SerializedBytes: n.serializeLeafWithCompressedCommitments(cBytes, c1Bytes, c2Bytes),
+				CommitmentBytes: serializedPoints[idx],
+				SerializedBytes: n.serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2Bytes),
 			}
 			ret = append(ret, sn)
 			idx += 3
@@ -1557,25 +1560,25 @@ func (n *InternalNode) collectNonHashedNodes(list []VerkleNode, paths [][]byte,
 }
 
 // unpack one compressed commitment from the list of batch-compressed commitments
-func (n *InternalNode) serializeInternalWithCompressedCommitment(compressedPointsIdxs map[VerkleNode]int, compressedPoints [][32]byte) ([]byte, error) {
-	serialized := make([]byte, nodeTypeSize+bitlistSize+SerializedPointCompressedSize)
-	bitlist := serialized[internalBitlistOffset:internalNodeChildrenOffset]
+func (n *InternalNode) serializeInternalWithUncompressedCommitment(pointsIdx map[VerkleNode]int, serializedPoints [][SerializedPointUncompressedSize]byte) ([]byte, error) {
+	serialized := make([]byte, nodeTypeSize+bitlistSize+SerializedPointUncompressedSize)
+	bitlist := serialized[internalBitlistOffset:internalCommitmentOffset]
 	for i, c := range n.children {
 		if _, ok := c.(Empty); !ok {
 			setBit(bitlist, i)
 		}
 	}
 	serialized[nodeTypeOffset] = internalRLPType
-	pointidx, ok := compressedPointsIdxs[n]
+	pointidx, ok := pointsIdx[n]
 	if !ok {
 		return nil, fmt.Errorf("child node not found in cache")
 	}
-	copy(serialized[internalNodeChildrenOffset:], compressedPoints[pointidx][:])
+	copy(serialized[internalCommitmentOffset:], serializedPoints[pointidx][:])
 
 	return serialized, nil
 }
 
-func (n *LeafNode) serializeLeafWithCompressedCommitments(cBytes, c1Bytes, c2Bytes [32]byte) []byte {
+func (n *LeafNode) serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2Bytes [SerializedPointUncompressedSize]byte) []byte {
 	// Empty value in LeafNode used for padding.
 	var emptyValue [LeafValueSize]byte
 
@@ -1593,15 +1596,14 @@ func (n *LeafNode) serializeLeafWithCompressedCommitments(cBytes, c1Bytes, c2Byt
 	}
 
 	// Create the serialization.
-	baseSize := nodeTypeSize + StemSize + bitlistSize + 3*SerializedPointCompressedSize
-	result := make([]byte, baseSize, baseSize+4*32) // Extra pre-allocated capacity for 4 values.
+	result := make([]byte, nodeTypeSize+StemSize+bitlistSize+3*SerializedPointUncompressedSize+len(children))
 	result[0] = leafRLPType
 	copy(result[leafSteamOffset:], n.stem[:StemSize])
 	copy(result[leafBitlistOffset:], bitlist[:])
 	copy(result[leafCommitmentOffset:], cBytes[:])
 	copy(result[leafC1CommitmentOffset:], c1Bytes[:])
 	copy(result[leafC2CommitmentOffset:], c2Bytes[:])
-	result = append(result, children...)
+	copy(result[leafChildrenOffset:], children)
 
 	return result
 }

tree_test.go

@@ -582,7 +582,7 @@ func TestNodeSerde(t *testing.T) {
 	tree := New()
 	tree.Insert(zeroKeyTest, testValue, nil)
 	tree.Insert(fourtyKeyTest, testValue, nil)
-	origComm := tree.Commit().Bytes()
+	origComm := tree.Commit().BytesUncompressed()
 	root := tree.(*InternalNode)
 
 	// Serialize all the nodes
@@ -626,10 +626,10 @@ func TestNodeSerde(t *testing.T) {
 	resRoot.children[64] = resLeaf64
 
 	if !isInternalEqual(root, resRoot) {
-		t.Fatalf("parsed node not equal, %x != %x", root.commitment.Bytes(), resRoot.commitment.Bytes())
+		t.Fatalf("parsed node not equal, %x != %x", root.commitment.BytesUncompressed(), resRoot.commitment.BytesUncompressed())
 	}
 
-	if resRoot.Commitment().Bytes() != origComm {
+	if resRoot.Commitment().BytesUncompressed() != origComm {
 		t.Fatal("invalid deserialized commitment")
 	}
 }