Proof of absence border case test (#413)

* proof: add proof of absence border case tests

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* Proof of absence and presence for the same path with nil proof of presence value (#414)

* keep the extension statuses of absent stems

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* more improvements and tests

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* fix & extra test case

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* make steams and extStatuses be 1:1

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

* cleanup

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

---------

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

---------

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>

proof_ipa.go

@@ -398,8 +398,9 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { //
 			stems = append(stems, k[:31])
 		}
 	}
-	stemIndex := 0
-
+	if len(stems) != len(proof.ExtStatus) {
+		return nil, fmt.Errorf("invalid number of stems and extension statuses: %d != %d", len(stems), len(proof.ExtStatus))
+	}
 	var (
 		info  = map[string]stemInfo{}
 		paths [][]byte
@@ -412,81 +413,73 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { //
 		return nil, fmt.Errorf("proof of absence stems are not sorted")
 	}
 
-	// assign one or more stem to each stem info
+	// We build a cache of paths that have a presence extension status.
+	pathsWithExtPresent := map[string]struct{}{}
+	i := 0
 	for _, es := range proof.ExtStatus {
-		depth := es >> 3
-		path := stems[stemIndex][:depth]
+		if es&3 == extStatusPresent {
+			pathsWithExtPresent[string(stems[i][:es>>3])] = struct{}{}
+		}
+		i++
+	}
+
+	// assign one or more stem to each stem info
+	for i, es := range proof.ExtStatus {
 		si := stemInfo{
-			depth:    depth,
+			depth:    es >> 3,
 			stemType: es & 3,
 		}
+		path := stems[i][:si.depth]
 		switch si.stemType {
 		case extStatusAbsentEmpty:
 			// All keys that are part of a proof of absence, must contain empty
 			// prestate values. If that isn't the case, the proof is invalid.
-			for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search.
-				if bytes.HasPrefix(k, path) {
-					if proof.PreValues[i] != nil {
-						return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem)
-					}
+			for j := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil {
+					return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem)
 				}
 			}
 		case extStatusAbsentOther:
-			si.stem = poas[0]
-			poas = poas[1:]
 			// All keys that are part of a proof of absence, must contain empty
 			// prestate values. If that isn't the case, the proof is invalid.
-			for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search.
-				if bytes.HasPrefix(k, si.stem) {
-					if proof.PreValues[i] != nil {
-						return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem)
-					}
+			for j := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil {
+					return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem)
 				}
 			}
-		default:
-			// the first stem could be missing (e.g. the second stem in the
-			// group is the one that is present. Compare each key to the first
-			// stem, along the length of the path only.
-			stemPath := stems[stemIndex][:len(path)]
+
+			// For this absent path, we must first check if this path contains a proof of presence.
+			// If that is the case, we don't have to do anything since the corresponding leaf will be
+			// constructed by that extension status (already processed or to be processed).
+			// In other case, we should get the stem from the list of proof of absence stems.
+			if _, ok := pathsWithExtPresent[string(path)]; ok {
+				continue
+			}
+
+			// Note that this path doesn't have proof of presence (previous if check above), but
+			// it can have multiple proof of absence. If a previous proof of absence had already
+			// created the stemInfo for this path, we don't have to do anything.
+			if _, ok := info[string(path)]; ok {
+				continue
+			}
+
+			si.stem = poas[0]
+			poas = poas[1:]
+		case extStatusPresent:
 			si.values = map[byte][]byte{}
-			for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search.
-				if bytes.Equal(k[:len(path)], stemPath) && proof.PreValues[i] != nil {
-					si.values[k[31]] = proof.PreValues[i]
+			si.stem = stems[i]
+			for j, k := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.Equal(k[:31], si.stem) {
+					si.values[k[31]] = proof.PreValues[j]
 					si.has_c1 = si.has_c1 || (k[31] < 128)
 					si.has_c2 = si.has_c2 || (k[31] >= 128)
-					// This key has values, its stem is the one that
-					// is present.
-					if si.stem == nil {
-						si.stem = k[:31]
-						continue
-					}
-					// Any other key with values must have the same
-					// same previously detected stem. If that isn't the case,
-					// the proof is invalid.
-					if !bytes.Equal(si.stem, k[:31]) {
-						return nil, fmt.Errorf("multiple keys with values found for stem %x", k[:31])
-					}
 				}
 			}
-			// For a proof of presence, we must always have detected a stem.
-			// If that isn't the case, the proof is invalid.
-			if si.stem == nil {
-				return nil, fmt.Errorf("no stem found for path %x", path)
-			}
+		default:
+			return nil, fmt.Errorf("invalid extension status: %d", si.stemType)
 		}
 		info[string(path)] = si
 		paths = append(paths, path)
-
-		// Skip over all the stems that share the same path
-		// to the extension tree. This happens e.g. if two
-		// stems have the same path, but one is a proof of
-		// absence and the other one is present.
-		stemIndex++
-		for ; stemIndex < len(stems); stemIndex++ {
-			if !bytes.Equal(stems[stemIndex][:depth], path) {
-				break
-			}
-		}
 	}
 
 	if len(poas) != 0 {

proof_test.go

@@ -1016,6 +1016,47 @@ func TestProofOfAbsenceBorderCase(t *testing.T) {
 	}
 }
 
+func TestProofOfAbsenceBorderCaseReversed(t *testing.T) {
+	root := New()
+
+	key1, _ := hex.DecodeString("0001000000000000000000000000000000000000000000000000000000000001")
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+
+	// Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	// Generate a proof for the following keys:
+	// - key1, which is present.
+	// - key2, which isn't present.
+	// Note that all three keys will land on the same leaf value.
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key1, key2}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+}
+
 func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) {
 	t.Parallel()
 
@@ -1088,3 +1129,91 @@ func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) {
 		}
 	}
 }
+
+func TestProofOfPresenceWithEmptyValue(t *testing.T) {
+	root := New()
+
+	key1, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+
+	// Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000002")
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+}
+
+func TestDoubleProofOfAbsence(t *testing.T) {
+	root := New()
+
+	// Insert some keys.
+	key11, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+	key12, _ := hex.DecodeString("0003000000000000000000000000000000000000000000000000000000000001")
+
+	if err := root.Insert(key11, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+	if err := root.Insert(key12, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	// Try to prove to different stems that end up in the same LeafNode without any other proof of presence
+	// in that leaf node. i.e: two proof of absence in the same leaf node with no proof of presence.
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100")
+	key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000200")
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	// Depite we have two proof of absences for different steams, we should only have one
+	// stem in `others`. i.e: we only need one for both steams.
+	if len(proof.PoaStems) != 1 {
+		t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems))
+	}
+
+	// We need one extension status for each stem.
+	if len(proof.ExtStatus) != 2 {
+		t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems))
+	}
+}

tree.go

@@ -1464,34 +1464,39 @@ func (n *LeafNode) GetProofItems(keys keylist, _ NodeResolverFn) (*ProofElements
 		pe.Fis = append(pe.Fis, poly[:])
 	}
 
+	addedAbsentStems := map[string]struct{}{}
+
 	// Second pass: add the cn-level elements
 	for _, key := range keys {
 		pe.ByPath[string(key[:n.depth])] = n.commitment
 
 		// Proof of absence: case of a differing stem.
-		// Add an unopened stem-level node.
 		if !equalPaths(n.stem, key) {
-			// Corner case: don't add the poa stem if it's
-			// already present as a proof-of-absence for a
-			// different key, or for the same key (case of
-			// multiple missing keys being absent).
-			// The list of extension statuses has to be of
-			// length 1 at this level, so skip otherwise.
+			// If this is the first extension status added for this path,
+			// add the proof of absence stem (only once). If later we detect a proof of
+			// presence, we'll clear the list since that proof of presence
+			// will be enough to provide the stem.
 			if len(esses) == 0 {
-				esses = append(esses, extStatusAbsentOther|(n.depth<<3))
 				poass = append(poass, n.stem)
 			}
+			// Add an extension status absent other for this stem.
+			// Note we keep a cache to avoid adding the same stem twice (or more) if
+			// there're multiple keys with the same stem.
+			if _, ok := addedAbsentStems[string(key[:StemSize])]; !ok {
+				esses = append(esses, extStatusAbsentOther|(n.depth<<3))
+				addedAbsentStems[string(key[:StemSize])] = struct{}{}
+			}
 			pe.Vals = append(pe.Vals, nil)
 			continue
 		}
 
-		// corner case (see previous corner case): if a proof-of-absence
-		// stem was found, and it now turns out the same stem is used as
-		// a proof of presence, clear the proof-of-absence list to avoid
-		// redundancy.
+		// As mentioned above, if a proof-of-absence stem was found, and
+		// it now turns out the same stem is used as a proof of presence,
+		// clear the proof-of-absence list to avoid redundancy. Note that
+		// we don't delete the extension statuse since that is needed to
+		// figure out which is the correct stem for this path.
 		if len(poass) > 0 {
 			poass = nil
-			esses = nil
 		}
 
 		var (

tree_test.go

@@ -1050,8 +1050,8 @@ func TestGetProofItemsNoPoaIfStemPresent(t *testing.T) {
 	if len(poas) != 0 {
 		t.Fatalf("returned %d poas instead of 0", len(poas))
 	}
-	if len(esses) != 1 {
-		t.Fatalf("returned %d extension statuses instead of the expected 1", len(esses))
+	if len(esses) != 3 {
+		t.Fatalf("returned %d extension statuses instead of the expected 3", len(esses))
 	}
 }