From 51d310d4d687deb9d95d88aeae6a66fe10496ff6 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 13:17:29 -0500
Subject: [PATCH 01/30] types: erasure namespaces

---
 blockchain/msgs_test.go | 2 +-
 types/block.go          | 6 ++++--
 types/consts.go         | 2 +-
 types/shares.go         | 8 ++++++++
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
index 818c4cb2cb..6f0a828086 100644
--- a/blockchain/msgs_test.go
+++ b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a2016106406aededa633a265efd5833af53775af5a27c803518313c9b77e68875926a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c10a30ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07123000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c11230ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20a50c5aca3e03471f1aad5767d39bcd68b62c493b056fb8f39a7080a72b1322d06a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7350a30ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f123000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7351230ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
diff --git a/types/block.go b/types/block.go
index d442ad86a9..3b48ef9b5e 100644
--- a/types/block.go
+++ b/types/block.go
@@ -201,7 +201,7 @@ func (b *Block) fillHeader() {
 // that are a function of the block data.
 func (b *Block) fillDataAvailabilityHeader() {
 	namespacedShares := b.Data.computeShares()
-	shares := namespacedShares.RawShares()
+	shares := namespacedShares.NamedShares()
 	if len(shares) == 0 {
 		// no shares -> no row/colum roots -> hash(empty)
 		b.DataHash = b.DataAvailabilityHeader.Hash()
@@ -264,6 +264,8 @@ func mustPush(rowTree *nmt.NamespacedMerkleTree, id namespace.ID, data []byte) {
 	}
 }
 
+// the erasured leaves are going to be longer than the non erasured leaves.
+
 // PutBlock posts and pins erasured block data to IPFS using the provided
 // ipld.NodeAdder. Note: the erasured data is currently recomputed
 func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error {
@@ -273,7 +275,7 @@ func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error
 
 	// recompute the shares
 	namespacedShares := b.Data.computeShares()
-	shares := namespacedShares.RawShares()
+	shares := namespacedShares.NamedShares()
 
 	// don't do anything if there is no data to put on IPFS
 	if len(shares) == 0 {
diff --git a/types/consts.go b/types/consts.go
index 78872f9a1a..1352c8691e 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -11,7 +11,7 @@ import (
 const (
 	// ShareSize is the size of a share (in bytes).
 	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-	ShareSize = 256
+	ShareSize = 248
 
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
diff --git a/types/shares.go b/types/shares.go
index 71f41a1f9d..58593ce640 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -38,6 +38,14 @@ func (ns NamespacedShares) RawShares() [][]byte {
 	return res
 }
 
+func (ns NamespacedShares) NamedShares() [][]byte {
+	res := make([][]byte, len(ns))
+	for i, nsh := range ns {
+		res[i] = append(nsh.ID, nsh.Share...)
+	}
+	return res
+}
+
 func (tx Tx) MarshalDelimited() ([]byte, error) {
 	lenBuf := make([]byte, binary.MaxVarintLen64)
 	length := uint64(len(tx))

From ddbaa9c8bbcd83ec06ec82462470ebf947ba6834 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 13:37:44 -0500
Subject: [PATCH 02/30] use methods instead of fields

---
 types/shares.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/types/shares.go b/types/shares.go
index 58593ce640..6789692236 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -41,7 +41,7 @@ func (ns NamespacedShares) RawShares() [][]byte {
 func (ns NamespacedShares) NamedShares() [][]byte {
 	res := make([][]byte, len(ns))
 	for i, nsh := range ns {
-		res[i] = append(nsh.ID, nsh.Share...)
+		res[i] = append(nsh.NamespaceID(), nsh.Data()...)
 	}
 	return res
 }

From ed06600b86af972b9e445b18923f9311c4803fca Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 14:03:19 -0500
Subject: [PATCH 03/30] remove unnecessary compteShares call

---
 types/block_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/types/block_test.go b/types/block_test.go
index 87f42479b3..f6675b8290 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1360,7 +1360,6 @@ func TestPutBlock(t *testing.T) {
 			defer cancel()
 
 			block.fillDataAvailabilityHeader()
-			tc.blockData.computeShares()
 			for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() {
 				// recreate the cids using only the computed roots
 				cid, err := nodes.CidFromNamespacedSha256(rowRoot)
@@ -1387,6 +1386,10 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
+func TestBlockRecovery(t *testing.T) {
+
+}
+
 func generateRandomData(msgCount int) Data {
 	out := make([]Message, msgCount)
 	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, ShareSize) {

From 1ff8b305986e04ee3df247014e355dc8682290a5 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 16:27:10 -0500
Subject: [PATCH 04/30] test block recovery

---
 types/block.go      |  36 ++++++++++++---
 types/block_test.go | 110 +++++++++++++++++++++++++++++++++++++++++++-
 types/consts.go     |   6 ++-
 3 files changed, 143 insertions(+), 9 deletions(-)

diff --git a/types/block.go b/types/block.go
index 3b48ef9b5e..8774b5d4df 100644
--- a/types/block.go
+++ b/types/block.go
@@ -6,7 +6,6 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"math"
 	"strings"
 	"time"
 
@@ -217,6 +216,7 @@ func (b *Block) fillDataAvailabilityHeader() {
 
 	// record the widths
 	squareWidth := extendedDataSquare.Width()
+	fmt.Println("square width", squareWidth)
 
 	b.DataAvailabilityHeader = DataAvailabilityHeader{
 		RowsRoots:   make([]namespace.IntervalDigest, squareWidth),
@@ -1348,12 +1348,12 @@ func (data *Data) computeShares() NamespacedShares {
 	evidenceShares := data.Evidence.splitIntoShares(ShareSize)
 
 	// application data shares from messages:
-	msgShares := data.Messages.splitIntoShares(ShareSize)
+	msgShares := data.Messages.splitIntoShares(AdjustedShareSize)
 	curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares)
 
 	// FIXME(ismail): this is not a power of two
 	// see: https://github.com/lazyledger/lazyledger-specs/issues/80 and
-	wantLen := getNextSquareNum(curLen)
+	wantLen := NextPowerOf2(curLen)
 	tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize)
 
 	return append(append(append(append(
@@ -1364,10 +1364,32 @@ func (data *Data) computeShares() NamespacedShares {
 		tailShares...)
 }
 
-func getNextSquareNum(length int) int {
-	width := int(math.Ceil(math.Sqrt(float64(length))))
-	// TODO(ismail): make width a power of two instead
-	return width * width
+// NextPowerOf2 returns the next lowest power of 2 unless the input is a power
+// of two, in which case it returns the input
+func NextPowerOf2(v int) int {
+	if v == 1 {
+		return 1
+	}
+	// keep track of the input
+	i := v
+
+	// find the next highest power using bit mashing
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+
+	// check if the input was the next highest power
+	if i == v {
+		return v
+	}
+
+	// return the next lowest power
+	return v / 2
 }
 
 type Message struct {
diff --git a/types/block_test.go b/types/block_test.go
index f6675b8290..2025280c08 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -5,9 +5,10 @@ import (
 	// number generator here and we can run the tests a bit faster
 	stdbytes "bytes"
 	"context"
-	"crypto/rand"
 	"encoding/hex"
+	"fmt"
 	"math"
+	"math/rand"
 	"os"
 	"reflect"
 	"sort"
@@ -19,6 +20,7 @@ import (
 	coremock "github.com/ipfs/go-ipfs/core/mock"
 	"github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
+	"github.com/lazyledger/rsmt2d"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -1386,8 +1388,114 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
+func TestNextPowerOf2(t *testing.T) {
+	type test struct {
+		input    int
+		expected int
+	}
+	tests := []test{
+		{
+			input:    2,
+			expected: 2,
+		},
+		{
+			input:    11,
+			expected: 8,
+		},
+		{
+			input:    511,
+			expected: 256,
+		},
+		{
+			input:    1,
+			expected: 1,
+		},
+		{
+			input:    0,
+			expected: 0,
+		},
+	}
+	for _, tt := range tests {
+		res := NextPowerOf2(tt.input)
+		assert.Equal(t, tt.expected, res)
+	}
+}
+
+// this test should be moved where PutBlock gets moved.
 func TestBlockRecovery(t *testing.T) {
+	ipfsNode, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+
+	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
+	if err != nil {
+		t.Error(err)
+	}
+
+	testCases := []struct {
+		name      string
+		blockData Data
+		expectErr bool
+		errString string
+	}{
+		{"16 leaves", generateRandomData(16), false, ""},
+		{"max square size", generateRandomData(17), false, ""},
+	}
+	ctx := context.Background()
+	for _, tc := range testCases {
+		tc := tc
+
+		block := &Block{Data: tc.blockData}
+
+		t.Run(tc.name, func(t *testing.T) {
+			err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning())
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errString)
+				return
+			}
+
+			require.NoError(t, err)
 
+			block.fillDataAvailabilityHeader()
+
+			rowRoots := block.DataAvailabilityHeader.RowsRoots.Bytes()
+			colRoots := block.DataAvailabilityHeader.ColumnRoots.Bytes()
+
+			data := tc.blockData.computeShares().NamedShares()
+
+			for _, d := range data {
+				fmt.Println(tc.name, len(d), len(rowRoots))
+			}
+
+			_, err := rsmt2d.RepairExtendedDataSquare(
+				rowRoots,
+				colRoots,
+				removeRandShares(data),
+				rsmt2d.RSGF8,
+			)
+
+			require.NoError(t, err)
+			// perform some check that namespaces are recovered as well
+
+		})
+	}
+}
+
+func removeRandShares(data [][]byte) [][]byte {
+	count := len(data)
+	// remove half of the shares randomly
+	for i := 0; i < (count / 2); {
+		ind := rand.Intn(count)
+		if len(data[ind]) == 0 {
+			continue
+		}
+		data[ind] = nil
+		i++
+	}
+	fmt.Println("removal data len", len(data))
+	return data
 }
 
 func generateRandomData(msgCount int) Data {
diff --git a/types/consts.go b/types/consts.go
index 1352c8691e..cabda1ea70 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -11,11 +11,15 @@ import (
 const (
 	// ShareSize is the size of a share (in bytes).
 	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-	ShareSize = 248
+	ShareSize = 256
 
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
 
+	ShareReservedBytes = 1
+
+	AdjustedShareSize = ShareSize - NamespaceSize
+
 	// MaxSquareSize is the maximum number of
 	// rows/columns of the original data shares in square layout.
 	// Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec.

From a4c6eaf5c539092d0a15cc311ffe0586913d1dff Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 22:45:02 -0500
Subject: [PATCH 05/30] don't mess things up by using power of two

---
 blockchain/msgs_test.go |  2 +-
 types/block.go          | 35 +++++++----------------------------
 types/block_test.go     | 33 ---------------------------------
 3 files changed, 8 insertions(+), 62 deletions(-)

diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
index 6f0a828086..7b34b80c8a 100644
--- a/blockchain/msgs_test.go
+++ b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20a50c5aca3e03471f1aad5767d39bcd68b62c493b056fb8f39a7080a72b1322d06a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7350a30ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f123000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7351230ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20c8672c31352c3eb8fe750c6ecc537bbc5db78159dc51200a3874bda26de3676a6a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a30000000000000000100000000000000013afc90c2dff77c04cec04c3fef2184bfe08c3200de1a7b31adae09dabaa8bb2c0a30fffffffffffffffffffffffffffffffff628558cf5d81c556150b08788dd9746c295a82839182ff95ccddebcd0b9271d1230000000000000000100000000000000013afc90c2dff77c04cec04c3fef2184bfe08c3200de1a7b31adae09dabaa8bb2c1230fffffffffffffffffffffffffffffffff628558cf5d81c556150b08788dd9746c295a82839182ff95ccddebcd0b9271d"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
diff --git a/types/block.go b/types/block.go
index 8774b5d4df..d2ce7fb4fb 100644
--- a/types/block.go
+++ b/types/block.go
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
@@ -1348,12 +1349,12 @@ func (data *Data) computeShares() NamespacedShares {
 	evidenceShares := data.Evidence.splitIntoShares(ShareSize)
 
 	// application data shares from messages:
-	msgShares := data.Messages.splitIntoShares(AdjustedShareSize)
+	msgShares := data.Messages.splitIntoShares(AdjustedMessageSize)
 	curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares)
 
 	// FIXME(ismail): this is not a power of two
 	// see: https://github.com/lazyledger/lazyledger-specs/issues/80 and
-	wantLen := NextPowerOf2(curLen)
+	wantLen := getNextSquareNum(curLen)
 	tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize)
 
 	return append(append(append(append(
@@ -1364,32 +1365,10 @@ func (data *Data) computeShares() NamespacedShares {
 		tailShares...)
 }
 
-// NextPowerOf2 returns the next lowest power of 2 unless the input is a power
-// of two, in which case it returns the input
-func NextPowerOf2(v int) int {
-	if v == 1 {
-		return 1
-	}
-	// keep track of the input
-	i := v
-
-	// find the next highest power using bit mashing
-	v--
-	v |= v >> 1
-	v |= v >> 2
-	v |= v >> 4
-	v |= v >> 8
-	v |= v >> 16
-	v |= v >> 32
-	v++
-
-	// check if the input was the next highest power
-	if i == v {
-		return v
-	}
-
-	// return the next lowest power
-	return v / 2
+func getNextSquareNum(length int) int {
+	width := int(math.Ceil(math.Sqrt(float64(length))))
+	// TODO(ismail): make width a power of two instead
+	return width * width
 }
 
 type Message struct {
diff --git a/types/block_test.go b/types/block_test.go
index 2025280c08..c35bdaa62d 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1388,39 +1388,6 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
-func TestNextPowerOf2(t *testing.T) {
-	type test struct {
-		input    int
-		expected int
-	}
-	tests := []test{
-		{
-			input:    2,
-			expected: 2,
-		},
-		{
-			input:    11,
-			expected: 8,
-		},
-		{
-			input:    511,
-			expected: 256,
-		},
-		{
-			input:    1,
-			expected: 1,
-		},
-		{
-			input:    0,
-			expected: 0,
-		},
-	}
-	for _, tt := range tests {
-		res := NextPowerOf2(tt.input)
-		assert.Equal(t, tt.expected, res)
-	}
-}
-
 // this test should be moved where PutBlock gets moved.
 func TestBlockRecovery(t *testing.T) {
 	ipfsNode, err := coremock.NewMockNode()

From a2f2ba170bb34b2e547fea8bf217580205557e6b Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 22:45:26 -0500
Subject: [PATCH 06/30] use better const names

---
 types/consts.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/types/consts.go b/types/consts.go
index cabda1ea70..cf159f4927 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -16,9 +16,7 @@ const (
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
 
-	ShareReservedBytes = 1
-
-	AdjustedShareSize = ShareSize - NamespaceSize
+	AdjustedMessageSize = ShareSize - NamespaceSize
 
 	// MaxSquareSize is the maximum number of
 	// rows/columns of the original data shares in square layout.

From dd6d65f0499548658ad0de4a8572c4de0fc08b82 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 13:17:29 -0500
Subject: [PATCH 07/30] types: erasure namespaces

---
 blockchain/msgs_test.go | 2 +-
 types/block.go          | 6 ++++--
 types/consts.go         | 2 +-
 types/shares.go         | 8 ++++++++
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
index 818c4cb2cb..6f0a828086 100644
--- a/blockchain/msgs_test.go
+++ b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a2016106406aededa633a265efd5833af53775af5a27c803518313c9b77e68875926a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c10a30ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07123000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c11230ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20a50c5aca3e03471f1aad5767d39bcd68b62c493b056fb8f39a7080a72b1322d06a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7350a30ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f123000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7351230ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
diff --git a/types/block.go b/types/block.go
index d442ad86a9..3b48ef9b5e 100644
--- a/types/block.go
+++ b/types/block.go
@@ -201,7 +201,7 @@ func (b *Block) fillHeader() {
 // that are a function of the block data.
 func (b *Block) fillDataAvailabilityHeader() {
 	namespacedShares := b.Data.computeShares()
-	shares := namespacedShares.RawShares()
+	shares := namespacedShares.NamedShares()
 	if len(shares) == 0 {
 		// no shares -> no row/colum roots -> hash(empty)
 		b.DataHash = b.DataAvailabilityHeader.Hash()
@@ -264,6 +264,8 @@ func mustPush(rowTree *nmt.NamespacedMerkleTree, id namespace.ID, data []byte) {
 	}
 }
 
+// the erasured leaves are going to be longer than the non erasured leaves.
+
 // PutBlock posts and pins erasured block data to IPFS using the provided
 // ipld.NodeAdder. Note: the erasured data is currently recomputed
 func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error {
@@ -273,7 +275,7 @@ func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error
 
 	// recompute the shares
 	namespacedShares := b.Data.computeShares()
-	shares := namespacedShares.RawShares()
+	shares := namespacedShares.NamedShares()
 
 	// don't do anything if there is no data to put on IPFS
 	if len(shares) == 0 {
diff --git a/types/consts.go b/types/consts.go
index 78872f9a1a..1352c8691e 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -11,7 +11,7 @@ import (
 const (
 	// ShareSize is the size of a share (in bytes).
 	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-	ShareSize = 256
+	ShareSize = 248
 
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
diff --git a/types/shares.go b/types/shares.go
index 71f41a1f9d..58593ce640 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -38,6 +38,14 @@ func (ns NamespacedShares) RawShares() [][]byte {
 	return res
 }
 
+func (ns NamespacedShares) NamedShares() [][]byte {
+	res := make([][]byte, len(ns))
+	for i, nsh := range ns {
+		res[i] = append(nsh.ID, nsh.Share...)
+	}
+	return res
+}
+
 func (tx Tx) MarshalDelimited() ([]byte, error) {
 	lenBuf := make([]byte, binary.MaxVarintLen64)
 	length := uint64(len(tx))

From f3905243404ab444e5a14800b5964e6210108438 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 13:37:44 -0500
Subject: [PATCH 08/30] use methods instead of fields

---
 types/shares.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/types/shares.go b/types/shares.go
index 58593ce640..6789692236 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -41,7 +41,7 @@ func (ns NamespacedShares) RawShares() [][]byte {
 func (ns NamespacedShares) NamedShares() [][]byte {
 	res := make([][]byte, len(ns))
 	for i, nsh := range ns {
-		res[i] = append(nsh.ID, nsh.Share...)
+		res[i] = append(nsh.NamespaceID(), nsh.Data()...)
 	}
 	return res
 }

From 43ad1faeb3cb22db12d3b371597f669ccf0132d5 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 14:03:19 -0500
Subject: [PATCH 09/30] remove unnecessary compteShares call

---
 types/block_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/types/block_test.go b/types/block_test.go
index 87f42479b3..f6675b8290 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1360,7 +1360,6 @@ func TestPutBlock(t *testing.T) {
 			defer cancel()
 
 			block.fillDataAvailabilityHeader()
-			tc.blockData.computeShares()
 			for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() {
 				// recreate the cids using only the computed roots
 				cid, err := nodes.CidFromNamespacedSha256(rowRoot)
@@ -1387,6 +1386,10 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
+func TestBlockRecovery(t *testing.T) {
+
+}
+
 func generateRandomData(msgCount int) Data {
 	out := make([]Message, msgCount)
 	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, ShareSize) {

From 9bec23470b187362b761753a1a7e2e086e672f96 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 16:27:10 -0500
Subject: [PATCH 10/30] test block recovery

---
 types/block.go      |  36 ++++++++++++---
 types/block_test.go | 110 +++++++++++++++++++++++++++++++++++++++++++-
 types/consts.go     |   6 ++-
 3 files changed, 143 insertions(+), 9 deletions(-)

diff --git a/types/block.go b/types/block.go
index 3b48ef9b5e..8774b5d4df 100644
--- a/types/block.go
+++ b/types/block.go
@@ -6,7 +6,6 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"math"
 	"strings"
 	"time"
 
@@ -217,6 +216,7 @@ func (b *Block) fillDataAvailabilityHeader() {
 
 	// record the widths
 	squareWidth := extendedDataSquare.Width()
+	fmt.Println("square width", squareWidth)
 
 	b.DataAvailabilityHeader = DataAvailabilityHeader{
 		RowsRoots:   make([]namespace.IntervalDigest, squareWidth),
@@ -1348,12 +1348,12 @@ func (data *Data) computeShares() NamespacedShares {
 	evidenceShares := data.Evidence.splitIntoShares(ShareSize)
 
 	// application data shares from messages:
-	msgShares := data.Messages.splitIntoShares(ShareSize)
+	msgShares := data.Messages.splitIntoShares(AdjustedShareSize)
 	curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares)
 
 	// FIXME(ismail): this is not a power of two
 	// see: https://github.com/lazyledger/lazyledger-specs/issues/80 and
-	wantLen := getNextSquareNum(curLen)
+	wantLen := NextPowerOf2(curLen)
 	tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize)
 
 	return append(append(append(append(
@@ -1364,10 +1364,32 @@ func (data *Data) computeShares() NamespacedShares {
 		tailShares...)
 }
 
-func getNextSquareNum(length int) int {
-	width := int(math.Ceil(math.Sqrt(float64(length))))
-	// TODO(ismail): make width a power of two instead
-	return width * width
+// NextPowerOf2 returns the next lowest power of 2 unless the input is a power
+// of two, in which case it returns the input
+func NextPowerOf2(v int) int {
+	if v == 1 {
+		return 1
+	}
+	// keep track of the input
+	i := v
+
+	// find the next highest power using bit mashing
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+
+	// check if the input was the next highest power
+	if i == v {
+		return v
+	}
+
+	// return the next lowest power
+	return v / 2
 }
 
 type Message struct {
diff --git a/types/block_test.go b/types/block_test.go
index f6675b8290..2025280c08 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -5,9 +5,10 @@ import (
 	// number generator here and we can run the tests a bit faster
 	stdbytes "bytes"
 	"context"
-	"crypto/rand"
 	"encoding/hex"
+	"fmt"
 	"math"
+	"math/rand"
 	"os"
 	"reflect"
 	"sort"
@@ -19,6 +20,7 @@ import (
 	coremock "github.com/ipfs/go-ipfs/core/mock"
 	"github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
+	"github.com/lazyledger/rsmt2d"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -1386,8 +1388,114 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
+func TestNextPowerOf2(t *testing.T) {
+	type test struct {
+		input    int
+		expected int
+	}
+	tests := []test{
+		{
+			input:    2,
+			expected: 2,
+		},
+		{
+			input:    11,
+			expected: 8,
+		},
+		{
+			input:    511,
+			expected: 256,
+		},
+		{
+			input:    1,
+			expected: 1,
+		},
+		{
+			input:    0,
+			expected: 0,
+		},
+	}
+	for _, tt := range tests {
+		res := NextPowerOf2(tt.input)
+		assert.Equal(t, tt.expected, res)
+	}
+}
+
+// this test should be moved where PutBlock gets moved.
 func TestBlockRecovery(t *testing.T) {
+	ipfsNode, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+
+	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
+	if err != nil {
+		t.Error(err)
+	}
+
+	testCases := []struct {
+		name      string
+		blockData Data
+		expectErr bool
+		errString string
+	}{
+		{"16 leaves", generateRandomData(16), false, ""},
+		{"max square size", generateRandomData(17), false, ""},
+	}
+	ctx := context.Background()
+	for _, tc := range testCases {
+		tc := tc
+
+		block := &Block{Data: tc.blockData}
+
+		t.Run(tc.name, func(t *testing.T) {
+			err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning())
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errString)
+				return
+			}
+
+			require.NoError(t, err)
 
+			block.fillDataAvailabilityHeader()
+
+			rowRoots := block.DataAvailabilityHeader.RowsRoots.Bytes()
+			colRoots := block.DataAvailabilityHeader.ColumnRoots.Bytes()
+
+			data := tc.blockData.computeShares().NamedShares()
+
+			for _, d := range data {
+				fmt.Println(tc.name, len(d), len(rowRoots))
+			}
+
+			_, err := rsmt2d.RepairExtendedDataSquare(
+				rowRoots,
+				colRoots,
+				removeRandShares(data),
+				rsmt2d.RSGF8,
+			)
+
+			require.NoError(t, err)
+			// perform some check that namespaces are recovered as well
+
+		})
+	}
+}
+
+func removeRandShares(data [][]byte) [][]byte {
+	count := len(data)
+	// remove half of the shares randomly
+	for i := 0; i < (count / 2); {
+		ind := rand.Intn(count)
+		if len(data[ind]) == 0 {
+			continue
+		}
+		data[ind] = nil
+		i++
+	}
+	fmt.Println("removal data len", len(data))
+	return data
 }
 
 func generateRandomData(msgCount int) Data {
diff --git a/types/consts.go b/types/consts.go
index 1352c8691e..cabda1ea70 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -11,11 +11,15 @@ import (
 const (
 	// ShareSize is the size of a share (in bytes).
 	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-	ShareSize = 248
+	ShareSize = 256
 
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
 
+	ShareReservedBytes = 1
+
+	AdjustedShareSize = ShareSize - NamespaceSize
+
 	// MaxSquareSize is the maximum number of
 	// rows/columns of the original data shares in square layout.
 	// Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec.

From 3d45d8a6f3ff74ab7f0e3522a009c632efcdfa7c Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 22:45:02 -0500
Subject: [PATCH 11/30] don't mess things up by using power of two

---
 blockchain/msgs_test.go |  2 +-
 types/block.go          | 35 +++++++----------------------------
 types/block_test.go     | 33 ---------------------------------
 3 files changed, 8 insertions(+), 62 deletions(-)

diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
index 6f0a828086..7b34b80c8a 100644
--- a/blockchain/msgs_test.go
+++ b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20a50c5aca3e03471f1aad5767d39bcd68b62c493b056fb8f39a7080a72b1322d06a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7350a30ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f123000000000000000010000000000000001bcd316ec9f25c2953d3d5642465370ab8002887a0740d1403403c2970b6ea7351230ffffffffffffffffffffffffffffffffc241c2e7a47e25d21b2dfbd5089bdc442305da97a61fb3ada8489e89e84fce4f"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a20c8672c31352c3eb8fe750c6ecc537bbc5db78159dc51200a3874bda26de3676a6a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a30000000000000000100000000000000013afc90c2dff77c04cec04c3fef2184bfe08c3200de1a7b31adae09dabaa8bb2c0a30fffffffffffffffffffffffffffffffff628558cf5d81c556150b08788dd9746c295a82839182ff95ccddebcd0b9271d1230000000000000000100000000000000013afc90c2dff77c04cec04c3fef2184bfe08c3200de1a7b31adae09dabaa8bb2c1230fffffffffffffffffffffffffffffffff628558cf5d81c556150b08788dd9746c295a82839182ff95ccddebcd0b9271d"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
diff --git a/types/block.go b/types/block.go
index 8774b5d4df..d2ce7fb4fb 100644
--- a/types/block.go
+++ b/types/block.go
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
@@ -1348,12 +1349,12 @@ func (data *Data) computeShares() NamespacedShares {
 	evidenceShares := data.Evidence.splitIntoShares(ShareSize)
 
 	// application data shares from messages:
-	msgShares := data.Messages.splitIntoShares(AdjustedShareSize)
+	msgShares := data.Messages.splitIntoShares(AdjustedMessageSize)
 	curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares)
 
 	// FIXME(ismail): this is not a power of two
 	// see: https://github.com/lazyledger/lazyledger-specs/issues/80 and
-	wantLen := NextPowerOf2(curLen)
+	wantLen := getNextSquareNum(curLen)
 	tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize)
 
 	return append(append(append(append(
@@ -1364,32 +1365,10 @@ func (data *Data) computeShares() NamespacedShares {
 		tailShares...)
 }
 
-// NextPowerOf2 returns the next lowest power of 2 unless the input is a power
-// of two, in which case it returns the input
-func NextPowerOf2(v int) int {
-	if v == 1 {
-		return 1
-	}
-	// keep track of the input
-	i := v
-
-	// find the next highest power using bit mashing
-	v--
-	v |= v >> 1
-	v |= v >> 2
-	v |= v >> 4
-	v |= v >> 8
-	v |= v >> 16
-	v |= v >> 32
-	v++
-
-	// check if the input was the next highest power
-	if i == v {
-		return v
-	}
-
-	// return the next lowest power
-	return v / 2
+func getNextSquareNum(length int) int {
+	width := int(math.Ceil(math.Sqrt(float64(length))))
+	// TODO(ismail): make width a power of two instead
+	return width * width
 }
 
 type Message struct {
diff --git a/types/block_test.go b/types/block_test.go
index 2025280c08..c35bdaa62d 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1388,39 +1388,6 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
-func TestNextPowerOf2(t *testing.T) {
-	type test struct {
-		input    int
-		expected int
-	}
-	tests := []test{
-		{
-			input:    2,
-			expected: 2,
-		},
-		{
-			input:    11,
-			expected: 8,
-		},
-		{
-			input:    511,
-			expected: 256,
-		},
-		{
-			input:    1,
-			expected: 1,
-		},
-		{
-			input:    0,
-			expected: 0,
-		},
-	}
-	for _, tt := range tests {
-		res := NextPowerOf2(tt.input)
-		assert.Equal(t, tt.expected, res)
-	}
-}
-
 // this test should be moved where PutBlock gets moved.
 func TestBlockRecovery(t *testing.T) {
 	ipfsNode, err := coremock.NewMockNode()

From 964e8fc5241ad825ea60e83882190ca684af3509 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 22:45:26 -0500
Subject: [PATCH 12/30] use better const names

---
 types/consts.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/types/consts.go b/types/consts.go
index cabda1ea70..cf159f4927 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -16,9 +16,7 @@ const (
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
 
-	ShareReservedBytes = 1
-
-	AdjustedShareSize = ShareSize - NamespaceSize
+	AdjustedMessageSize = ShareSize - NamespaceSize
 
 	// MaxSquareSize is the maximum number of
 	// rows/columns of the original data shares in square layout.

From 3482ebbb23e8a5dcfd0367d34aa34f71a65baa74 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Thu, 18 Mar 2021 23:11:56 -0500
Subject: [PATCH 13/30] remove print statements

---
 types/block.go      | 1 -
 types/block_test.go | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/types/block.go b/types/block.go
index d2ce7fb4fb..b4748e3a6d 100644
--- a/types/block.go
+++ b/types/block.go
@@ -217,7 +217,6 @@ func (b *Block) fillDataAvailabilityHeader() {
 
 	// record the widths
 	squareWidth := extendedDataSquare.Width()
-	fmt.Println("square width", squareWidth)
 
 	b.DataAvailabilityHeader = DataAvailabilityHeader{
 		RowsRoots:   make([]namespace.IntervalDigest, squareWidth),
diff --git a/types/block_test.go b/types/block_test.go
index c35bdaa62d..8757439b06 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1432,10 +1432,6 @@ func TestBlockRecovery(t *testing.T) {
 
 			data := tc.blockData.computeShares().NamedShares()
 
-			for _, d := range data {
-				fmt.Println(tc.name, len(d), len(rowRoots))
-			}
-
 			_, err := rsmt2d.RepairExtendedDataSquare(
 				rowRoots,
 				colRoots,

From 4de5bb684c5c5e39865085386e35297a00e8100a Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Mon, 22 Mar 2021 13:50:12 -0500
Subject: [PATCH 14/30] add docs to AdjustedMessageSize

---
 types/consts.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/types/consts.go b/types/consts.go
index cf159f4927..9074ef4941 100644
--- a/types/consts.go
+++ b/types/consts.go
@@ -16,6 +16,8 @@ const (
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
 
+	// AdjustedMessageSize describes the numbers of bytes in a share that are
+	// not reserved for the namespace
 	AdjustedMessageSize = ShareSize - NamespaceSize
 
 	// MaxSquareSize is the maximum number of

From c8c46678da357170fc794631d49ba6523ff9bce1 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Mon, 22 Mar 2021 13:55:05 -0500
Subject: [PATCH 15/30] used the types.AdjustedMessageSize instead of the local

---
 p2p/ipld/nmt_wrapper_test.go | 4 ++--
 p2p/ipld/read.go             | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/p2p/ipld/nmt_wrapper_test.go b/p2p/ipld/nmt_wrapper_test.go
index 3053ad1597..6c23cff54f 100644
--- a/p2p/ipld/nmt_wrapper_test.go
+++ b/p2p/ipld/nmt_wrapper_test.go
@@ -36,7 +36,7 @@ func TestRootErasuredNamespacedMerkleTree(t *testing.T) {
 	// the case, because the ErasuredNamespacedMerkleTree should add namespaces
 	// to the second half of the tree
 	size := 16
-	data := generateRandNamespacedRawData(size, types.NamespaceSize, AdjustedMessageSize)
+	data := generateRandNamespacedRawData(size, types.NamespaceSize, types.AdjustedMessageSize)
 	n := NewErasuredNamespacedMerkleTree(uint64(16))
 	tree := n.Constructor()
 	nmtTree := nmt.New(sha256.New())
@@ -110,7 +110,7 @@ func generateErasuredData(t *testing.T, numLeaves int) [][]byte {
 	raw := generateRandNamespacedRawData(
 		numLeaves,
 		types.NamespaceSize,
-		AdjustedMessageSize,
+		types.AdjustedMessageSize,
 	)
 	erasuredData, err := rsmt2d.Encode(raw, rsmt2d.RSGF8)
 	if err != nil {
diff --git a/p2p/ipld/read.go b/p2p/ipld/read.go
index 99e2558880..6469b8dbfe 100644
--- a/p2p/ipld/read.go
+++ b/p2p/ipld/read.go
@@ -8,11 +8,6 @@ import (
 	"github.com/ipfs/go-cid"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/path"
-	"github.com/lazyledger/lazyledger-core/types"
-)
-
-const (
-	AdjustedMessageSize = types.ShareSize - types.NamespaceSize
 )
 
 // /////////////////////////////////////

From 772ae43cd108985322f07b533c4ce4f8ee9e3634 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 09:40:31 -0500
Subject: [PATCH 16/30] moving out consts to avoid import cycle

---
 p2p/ipld/plugin/nodes/nodes.go |  1 +
 types/consts/consts.go         | 45 ++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 types/consts/consts.go

diff --git a/p2p/ipld/plugin/nodes/nodes.go b/p2p/ipld/plugin/nodes/nodes.go
index e465c2361b..d60729d585 100644
--- a/p2p/ipld/plugin/nodes/nodes.go
+++ b/p2p/ipld/plugin/nodes/nodes.go
@@ -221,6 +221,7 @@ func NewNmtNodeAdder(ctx context.Context, batch *format.Batch) *NmtNodeAdder {
 // Visit can be inserted into an nmt tree to create ipld.Nodes while computing the root
 func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) {
 	cid := mustCidFromNamespacedSha256(hash)
+	fmt.Println("collecting cid", cid.String())
 	switch len(children) {
 	case 1:
 		n.batch.Add(n.ctx, nmtLeafNode{
diff --git a/types/consts/consts.go b/types/consts/consts.go
new file mode 100644
index 0000000000..8aa39462f2
--- /dev/null
+++ b/types/consts/consts.go
@@ -0,0 +1,45 @@
+package consts
+
+import (
+	"crypto/sha256"
+
+	"github.com/lazyledger/nmt/namespace"
+)
+
+// This contains all constants of:
+// https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
+const (
+	// ShareSize is the size of a share (in bytes).
+	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
+	ShareSize = 256
+
+	// NamespaceSize is the namespace size in bytes.
+	NamespaceSize = 8
+
+	// ShareReservedBytes is the reserved bytes for contiguous appends.
+	ShareReservedBytes = 1
+
+	// TxShareSize is the number of bytes usable for tx/evidence/ISR shares.
+	TxShareSize = ShareSize - NamespaceSize - ShareReservedBytes
+	// MsgShareSize is the number of bytes usable for message shares.
+	MsgShareSize = ShareSize - NamespaceSize
+
+	// MaxSquareSize is the maximum number of
+	// rows/columns of the original data shares in square layout.
+	// Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec.
+	// 128*128*256 = 4 Megabytes
+	// TODO(ismail): settle on a proper max square
+	MaxSquareSize = 128
+)
+
+var (
+	TxNamespaceID                     = namespace.ID{0, 0, 0, 0, 0, 0, 0, 1}
+	IntermediateStateRootsNamespaceID = namespace.ID{0, 0, 0, 0, 0, 0, 0, 2}
+	EvidenceNamespaceID               = namespace.ID{0, 0, 0, 0, 0, 0, 0, 3}
+
+	TailPaddingNamespaceID  = namespace.ID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
+	ParitySharesNamespaceID = namespace.ID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
+
+	// change accordingly if another hash.Hash should be used as a base hasher in the NMT:
+	newBaseHashFunc = sha256.New
+)

From 2724fe88fc5729e2e0859297997562910974730f Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 09:41:29 -0500
Subject: [PATCH 17/30] re applying stashed changes

---
 p2p/ipld/nmt_wrapper.go |  2 +
 p2p/ipld/read_test.go   | 95 ++++++++++++++++++++++++++++++++++++++++-
 types/block_test.go     | 12 +++---
 3 files changed, 102 insertions(+), 7 deletions(-)

diff --git a/p2p/ipld/nmt_wrapper.go b/p2p/ipld/nmt_wrapper.go
index 62abbb5958..9f420dd5fb 100644
--- a/p2p/ipld/nmt_wrapper.go
+++ b/p2p/ipld/nmt_wrapper.go
@@ -2,6 +2,7 @@ package ipld
 
 import (
 	"crypto/sha256"
+	"fmt"
 
 	"github.com/lazyledger/lazyledger-core/types"
 	"github.com/lazyledger/nmt"
@@ -45,6 +46,7 @@ func (w *ErasuredNamespacedMerkleTree) Push(data []byte, idx rsmt2d.SquareIndex)
 	// determine the namespace based on where in the tree we're pushing
 	nsID := make(namespace.ID, types.NamespaceSize)
 
+	fmt.Println("axis ", idx.Axis, idx.Cell)
 	if idx.Axis+1 > 2*uint(w.squareSize) || idx.Cell+1 > 2*uint(w.squareSize) {
 		panic("pushed past predetermined square size")
 	}
diff --git a/p2p/ipld/read_test.go b/p2p/ipld/read_test.go
index 014642bfd8..6619783bd5 100644
--- a/p2p/ipld/read_test.go
+++ b/p2p/ipld/read_test.go
@@ -3,8 +3,9 @@ package ipld
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
 	"crypto/sha256"
+	"fmt"
+	"math/rand"
 	"sort"
 	"strings"
 	"testing"
@@ -18,7 +19,9 @@ import (
 	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
 	"github.com/lazyledger/lazyledger-core/types"
 	"github.com/lazyledger/nmt"
+	"github.com/lazyledger/rsmt2d"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestLeafPath(t *testing.T) {
@@ -153,6 +156,71 @@ func TestGetLeafData(t *testing.T) {
 	}
 }
 
+func TestBlockRecovery(t *testing.T) {
+	ipfsNode, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+
+	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
+	if err != nil {
+		t.Error(err)
+	}
+
+	testCases := []struct {
+		name      string
+		blockData types.Data
+		expectErr bool
+		errString string
+	}{
+		{"16 leaves", generateRandomData(16), false, ""},
+		// {"max square size", generateRandomData(17), false, ""},
+	}
+	ctx := context.Background()
+	for _, tc := range testCases {
+		tc := tc
+
+		block := &types.Block{
+			Data:       tc.blockData,
+			LastCommit: &types.Commit{},
+		}
+
+		block.Hash()
+
+		t.Run(tc.name, func(t *testing.T) {
+			err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning())
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errString)
+				return
+			}
+
+			require.NoError(t, err)
+
+			rowRoots := block.DataAvailabilityHeader.RowsRoots.Bytes()
+			colRoots := block.DataAvailabilityHeader.ColumnRoots.Bytes()
+
+			data := tc.blockData.ComputeShares().RawShares()
+
+			fmt.Println("row roots", len(rowRoots))
+
+			tree := NewErasuredNamespacedMerkleTree(uint64(len(rowRoots) / 2))
+
+			_, err := rsmt2d.RepairExtendedDataSquare(
+				rowRoots,
+				colRoots,
+				removeRandShares(data),
+				rsmt2d.RSGF8,
+				tree.Constructor,
+			)
+
+			require.NoError(t, err)
+			// perform some check that namespaces are recovered as well
+
+		})
+	}
+}
+
 // nmtcommitment generates the nmt root of some namespaced data
 func createNmtTree(
 	ctx context.Context,
@@ -199,3 +267,28 @@ func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byt
 func sortByteArrays(src [][]byte) {
 	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
 }
+
+func generateRandomData(msgCount int) types.Data {
+	out := make([]types.Message, msgCount)
+	for i, msg := range generateRandNamespacedRawData(msgCount, types.NamespaceSize, types.ShareSize) {
+		out[i] = types.Message{NamespaceID: msg[:types.NamespaceSize], Data: msg[:types.NamespaceSize]}
+	}
+	return types.Data{
+		Messages: types.Messages{MessagesList: out},
+	}
+}
+
+func removeRandShares(data [][]byte) [][]byte {
+	count := len(data)
+	// remove half of the shares randomly
+	for i := 0; i < (count / 2); {
+		ind := rand.Intn(count)
+		if len(data[ind]) == 0 {
+			continue
+		}
+		data[ind] = nil
+		i++
+	}
+	fmt.Println("removal data len", len(data))
+	return data
+}
diff --git a/types/block_test.go b/types/block_test.go
index 23b69a7054..58054a6344 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -5,9 +5,9 @@ import (
 	// number generator here and we can run the tests a bit faster
 	stdbytes "bytes"
 	"context"
-	"crypto/rand"
 	"encoding/hex"
 	"math"
+	"math/rand"
 	"os"
 	"reflect"
 	"sort"
@@ -1335,10 +1335,10 @@ func TestPutBlock(t *testing.T) {
 		expectErr bool
 		errString string
 	}{
-		{"no leaves", generateRandomData(0), false, ""},
-		{"single leaf", generateRandomData(1), false, ""},
+		// {"no leaves", generateRandomData(0), false, ""},
+		// {"single leaf", generateRandomData(1), false, ""},
 		{"16 leaves", generateRandomData(16), false, ""},
-		{"max square size", generateRandomData(MaxSquareSize), false, ""},
+		// {"max square size", generateRandomData(MaxSquareSize), false, ""},
 	}
 	ctx := context.Background()
 	for _, tc := range testCases {
@@ -1360,7 +1360,7 @@ func TestPutBlock(t *testing.T) {
 			defer cancel()
 
 			block.fillDataAvailabilityHeader()
-			tc.blockData.ComputeShares()
+
 			for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() {
 				// recreate the cids using only the computed roots
 				cid, err := nodes.CidFromNamespacedSha256(rowRoot)
@@ -1390,7 +1390,7 @@ func TestPutBlock(t *testing.T) {
 func generateRandomData(msgCount int) Data {
 	out := make([]Message, msgCount)
 	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, ShareSize) {
-		out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[:NamespaceSize]}
+		out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[NamespaceSize:]}
 	}
 	return Data{
 		Messages: Messages{MessagesList: out},

From 3f63b028d1fa958f8af9f82989abb85d9ea74372 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 22:48:28 -0500
Subject: [PATCH 18/30] return new instances of the wrapper per call of the
 constructor

---
 p2p/ipld/nmt_wrapper.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/p2p/ipld/nmt_wrapper.go b/p2p/ipld/nmt_wrapper.go
index 9f420dd5fb..0d7b9ed18e 100644
--- a/p2p/ipld/nmt_wrapper.go
+++ b/p2p/ipld/nmt_wrapper.go
@@ -2,7 +2,6 @@ package ipld
 
 import (
 	"crypto/sha256"
-	"fmt"
 
 	"github.com/lazyledger/lazyledger-core/types"
 	"github.com/lazyledger/nmt"
@@ -27,14 +26,15 @@ type ErasuredNamespacedMerkleTree struct {
 
 // NewErasuredNamespacedMerkleTree issues a new ErasuredNamespacedMerkleTree
 func NewErasuredNamespacedMerkleTree(squareSize uint64, setters ...nmt.Option) ErasuredNamespacedMerkleTree {
-	return ErasuredNamespacedMerkleTree{squareSize: squareSize, options: setters}
+	tree := nmt.New(sha256.New(), setters...)
+	return ErasuredNamespacedMerkleTree{squareSize: squareSize, options: setters, tree: tree}
 }
 
 // Constructor acts as the rsmt2d.TreeConstructorFn for
 // ErasuredNamespacedMerkleTree
 func (w ErasuredNamespacedMerkleTree) Constructor() rsmt2d.Tree {
-	w.tree = nmt.New(sha256.New(), w.options...)
-	return &w
+	newTree := NewErasuredNamespacedMerkleTree(w.squareSize, w.options...)
+	return &newTree
 }
 
 // Push adds the provided data to the underlying NamespaceMerkleTree, and
@@ -46,7 +46,6 @@ func (w *ErasuredNamespacedMerkleTree) Push(data []byte, idx rsmt2d.SquareIndex)
 	// determine the namespace based on where in the tree we're pushing
 	nsID := make(namespace.ID, types.NamespaceSize)
 
-	fmt.Println("axis ", idx.Axis, idx.Cell)
 	if idx.Axis+1 > 2*uint(w.squareSize) || idx.Cell+1 > 2*uint(w.squareSize) {
 		panic("pushed past predetermined square size")
 	}

From 173ddd7eba866800652522a5e178ca4e2b0bab9e Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 22:49:29 -0500
Subject: [PATCH 19/30] implement tests for recovering data from a square using
 the new nmt wrapper

---
 p2p/ipld/read_test.go  | 111 ++++++++++++++++++++++-------------------
 types/consts/consts.go |  45 -----------------
 2 files changed, 59 insertions(+), 97 deletions(-)
 delete mode 100644 types/consts/consts.go

diff --git a/p2p/ipld/read_test.go b/p2p/ipld/read_test.go
index 6619783bd5..6a17bb2776 100644
--- a/p2p/ipld/read_test.go
+++ b/p2p/ipld/read_test.go
@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"fmt"
+	"math"
 	"math/rand"
 	"sort"
 	"strings"
@@ -157,70 +157,87 @@ func TestGetLeafData(t *testing.T) {
 }
 
 func TestBlockRecovery(t *testing.T) {
-	ipfsNode, err := coremock.NewMockNode()
-	if err != nil {
-		t.Error(err)
-	}
+	// adjustedLeafSize describes the size of a leaf that will not get split
+	adjustedLeafSize := types.MsgShareSize
 
-	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
-	if err != nil {
-		t.Error(err)
-	}
+	originalSquareWidth := 2
+	sharecount := originalSquareWidth * originalSquareWidth
+	extendedSquareWidth := originalSquareWidth * originalSquareWidth
+	extendedShareCount := extendedSquareWidth * extendedSquareWidth
+
+	// generate test data
+	quarterShares := generateRandNamespacedRawData(sharecount, types.NamespaceSize, adjustedLeafSize)
+	allShares := generateRandNamespacedRawData(sharecount, types.NamespaceSize, adjustedLeafSize)
 
 	testCases := []struct {
-		name      string
-		blockData types.Data
+		name string
+		// blockData types.Data
+		shares    [][]byte
 		expectErr bool
 		errString string
+		d         int // number of shares to delete
 	}{
-		{"16 leaves", generateRandomData(16), false, ""},
-		// {"max square size", generateRandomData(17), false, ""},
+		// missing more shares causes RepairExtendedDataSquare to hang see
+		// https://github.com/lazyledger/rsmt2d/issues/21
+		{"missing 1/4 shares", quarterShares, false, "", extendedShareCount / 4},
+		{"missing all but one shares", allShares, true, "failed to solve data square", extendedShareCount - 1},
 	}
-	ctx := context.Background()
 	for _, tc := range testCases {
 		tc := tc
 
-		block := &types.Block{
-			Data:       tc.blockData,
-			LastCommit: &types.Commit{},
-		}
-
-		block.Hash()
-
 		t.Run(tc.name, func(t *testing.T) {
-			err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning())
-			if tc.expectErr {
-				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.errString)
-				return
-			}
+			squareSize := uint64(math.Sqrt(float64(len(tc.shares))))
 
-			require.NoError(t, err)
+			// create trees for creating roots
+			tree := NewErasuredNamespacedMerkleTree(squareSize)
+			recoverTree := NewErasuredNamespacedMerkleTree(squareSize)
 
-			rowRoots := block.DataAvailabilityHeader.RowsRoots.Bytes()
-			colRoots := block.DataAvailabilityHeader.ColumnRoots.Bytes()
-
-			data := tc.blockData.ComputeShares().RawShares()
+			eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.RSGF8, tree.Constructor)
+			if err != nil {
+				t.Error(err)
+			}
 
-			fmt.Println("row roots", len(rowRoots))
+			// calculate roots using the first complete square
+			rowRoots := eds.RowRoots()
+			colRoots := eds.ColumnRoots()
 
-			tree := NewErasuredNamespacedMerkleTree(uint64(len(rowRoots) / 2))
+			flat := flatten(eds)
 
-			_, err := rsmt2d.RepairExtendedDataSquare(
+			// recover a partially complete square
+			reds, err := rsmt2d.RepairExtendedDataSquare(
 				rowRoots,
 				colRoots,
-				removeRandShares(data),
+				removeRandShares(flat, tc.d),
 				rsmt2d.RSGF8,
-				tree.Constructor,
+				recoverTree.Constructor,
 			)
 
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errString)
+				return
+			}
+
 			require.NoError(t, err)
-			// perform some check that namespaces are recovered as well
 
+			// check that the squares are equal
+			assert.Equal(t, flatten(eds), flatten(reds))
 		})
 	}
 }
 
+func flatten(eds *rsmt2d.ExtendedDataSquare) [][]byte {
+	out := make([][]byte, eds.Width()*eds.Width())
+	count := 0
+	for i := uint(0); i < eds.Width(); i++ {
+		for _, share := range eds.Row(i) {
+			out[count] = share
+			count++
+		}
+	}
+	return out
+}
+
 // nmtcommitment generates the nmt root of some namespaced data
 func createNmtTree(
 	ctx context.Context,
@@ -268,20 +285,11 @@ func sortByteArrays(src [][]byte) {
 	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
 }
 
-func generateRandomData(msgCount int) types.Data {
-	out := make([]types.Message, msgCount)
-	for i, msg := range generateRandNamespacedRawData(msgCount, types.NamespaceSize, types.ShareSize) {
-		out[i] = types.Message{NamespaceID: msg[:types.NamespaceSize], Data: msg[:types.NamespaceSize]}
-	}
-	return types.Data{
-		Messages: types.Messages{MessagesList: out},
-	}
-}
-
-func removeRandShares(data [][]byte) [][]byte {
+// removes d shares from data
+func removeRandShares(data [][]byte, d int) [][]byte {
 	count := len(data)
-	// remove half of the shares randomly
-	for i := 0; i < (count / 2); {
+	// remove shares randomly
+	for i := 0; i < d; {
 		ind := rand.Intn(count)
 		if len(data[ind]) == 0 {
 			continue
@@ -289,6 +297,5 @@ func removeRandShares(data [][]byte) [][]byte {
 		data[ind] = nil
 		i++
 	}
-	fmt.Println("removal data len", len(data))
 	return data
 }
diff --git a/types/consts/consts.go b/types/consts/consts.go
deleted file mode 100644
index 8aa39462f2..0000000000
--- a/types/consts/consts.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package consts
-
-import (
-	"crypto/sha256"
-
-	"github.com/lazyledger/nmt/namespace"
-)
-
-// This contains all constants of:
-// https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-const (
-	// ShareSize is the size of a share (in bytes).
-	// see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/consensus.md#constants
-	ShareSize = 256
-
-	// NamespaceSize is the namespace size in bytes.
-	NamespaceSize = 8
-
-	// ShareReservedBytes is the reserved bytes for contiguous appends.
-	ShareReservedBytes = 1
-
-	// TxShareSize is the number of bytes usable for tx/evidence/ISR shares.
-	TxShareSize = ShareSize - NamespaceSize - ShareReservedBytes
-	// MsgShareSize is the number of bytes usable for message shares.
-	MsgShareSize = ShareSize - NamespaceSize
-
-	// MaxSquareSize is the maximum number of
-	// rows/columns of the original data shares in square layout.
-	// Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec.
-	// 128*128*256 = 4 Megabytes
-	// TODO(ismail): settle on a proper max square
-	MaxSquareSize = 128
-)
-
-var (
-	TxNamespaceID                     = namespace.ID{0, 0, 0, 0, 0, 0, 0, 1}
-	IntermediateStateRootsNamespaceID = namespace.ID{0, 0, 0, 0, 0, 0, 0, 2}
-	EvidenceNamespaceID               = namespace.ID{0, 0, 0, 0, 0, 0, 0, 3}
-
-	TailPaddingNamespaceID  = namespace.ID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
-	ParitySharesNamespaceID = namespace.ID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
-
-	// change accordingly if another hash.Hash should be used as a base hasher in the NMT:
-	newBaseHashFunc = sha256.New
-)

From e4e0c74bfc291f030767f0544fd6c939e9409a1a Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 22:50:58 -0500
Subject: [PATCH 20/30] fix sneaky append bug

---
 p2p/ipld/plugin/nodes/nodes.go |  1 -
 types/block_test.go            |  8 ++++----
 types/shares.go                | 12 +++++++++---
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/p2p/ipld/plugin/nodes/nodes.go b/p2p/ipld/plugin/nodes/nodes.go
index d60729d585..e465c2361b 100644
--- a/p2p/ipld/plugin/nodes/nodes.go
+++ b/p2p/ipld/plugin/nodes/nodes.go
@@ -221,7 +221,6 @@ func NewNmtNodeAdder(ctx context.Context, batch *format.Batch) *NmtNodeAdder {
 // Visit can be inserted into an nmt tree to create ipld.Nodes while computing the root
 func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) {
 	cid := mustCidFromNamespacedSha256(hash)
-	fmt.Println("collecting cid", cid.String())
 	switch len(children) {
 	case 1:
 		n.batch.Add(n.ctx, nmtLeafNode{
diff --git a/types/block_test.go b/types/block_test.go
index 58054a6344..559b839672 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1335,10 +1335,10 @@ func TestPutBlock(t *testing.T) {
 		expectErr bool
 		errString string
 	}{
-		// {"no leaves", generateRandomData(0), false, ""},
-		// {"single leaf", generateRandomData(1), false, ""},
+		{"no leaves", generateRandomData(0), false, ""},
+		{"single leaf", generateRandomData(1), false, ""},
 		{"16 leaves", generateRandomData(16), false, ""},
-		// {"max square size", generateRandomData(MaxSquareSize), false, ""},
+		{"max square size", generateRandomData(MaxSquareSize), false, ""},
 	}
 	ctx := context.Background()
 	for _, tc := range testCases {
@@ -1389,7 +1389,7 @@ func TestPutBlock(t *testing.T) {
 
 func generateRandomData(msgCount int) Data {
 	out := make([]Message, msgCount)
-	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, ShareSize) {
+	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, MsgShareSize-2) {
 		out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[NamespaceSize:]}
 	}
 	return Data{
diff --git a/types/shares.go b/types/shares.go
index 2611921edb..4abbf0d9bb 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -52,13 +52,17 @@ func (m Message) MarshalDelimited() ([]byte, error) {
 	lenBuf := make([]byte, binary.MaxVarintLen64)
 	length := uint64(len(m.Data))
 	n := binary.PutUvarint(lenBuf, length)
-
 	return append(lenBuf[:n], m.Data...), nil
 }
 
+// m.Data is being altered somehwere along the line, where the first two bytes are being replaces with the above append
+// the append isn't inherently wrong, it's that the data should not altered after that.
+
 // appendToShares appends raw data as shares.
 // Used for messages.
-func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
+func appendToShares(shares []NamespacedShare, id namespace.ID, rawData []byte) []NamespacedShare {
+	nid := make([]byte, len(id))
+	copy(nid, id)
 	if len(rawData) <= MsgShareSize {
 		rawShare := []byte(append(nid, rawData...))
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
@@ -72,7 +76,7 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte)
 
 // splitContiguous splits multiple raw data contiguously as shares.
 // Used for transactions, intermediate state roots, and evidence.
-func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
+func splitContiguous(id namespace.ID, rawDatas [][]byte) []NamespacedShare {
 	shares := make([]NamespacedShare, 0)
 	// Index into the outer slice of rawDatas
 	outerIndex := 0
@@ -80,6 +84,8 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
 	innerIndex := 0
 	for outerIndex < len(rawDatas) {
 		var rawData []byte
+		nid := make([]byte, len(id))
+		copy(nid, id)
 		startIndex := 0
 		rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
 		rawShare := []byte(append(append(nid, byte(startIndex)), rawData...))

From b1bc22df9e85cf1300cf059e0fd6fb6a3b97ffc4 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:10:41 -0500
Subject: [PATCH 21/30] remove moved test

---
 types/block_test.go | 73 ---------------------------------------------
 1 file changed, 73 deletions(-)

diff --git a/types/block_test.go b/types/block_test.go
index 521ba57325..1b69028b49 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1388,79 +1388,6 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
-// this test should be moved where PutBlock gets moved.
-func TestBlockRecovery(t *testing.T) {
-	ipfsNode, err := coremock.NewMockNode()
-	if err != nil {
-		t.Error(err)
-	}
-
-	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
-	if err != nil {
-		t.Error(err)
-	}
-
-	testCases := []struct {
-		name      string
-		blockData Data
-		expectErr bool
-		errString string
-	}{
-		{"16 leaves", generateRandomData(16), false, ""},
-		{"max square size", generateRandomData(17), false, ""},
-	}
-	ctx := context.Background()
-	for _, tc := range testCases {
-		tc := tc
-
-		block := &Block{Data: tc.blockData}
-
-		t.Run(tc.name, func(t *testing.T) {
-			err = block.PutBlock(ctx, ipfsAPI.Dag().Pinning())
-			if tc.expectErr {
-				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.errString)
-				return
-			}
-
-			require.NoError(t, err)
-
-			block.fillDataAvailabilityHeader()
-
-			rowRoots := block.DataAvailabilityHeader.RowsRoots.Bytes()
-			colRoots := block.DataAvailabilityHeader.ColumnRoots.Bytes()
-
-			data := tc.blockData.computeShares().NamedShares()
-
-			_, err := rsmt2d.RepairExtendedDataSquare(
-				rowRoots,
-				colRoots,
-				removeRandShares(data),
-				rsmt2d.RSGF8,
-			)
-
-			require.NoError(t, err)
-			// perform some check that namespaces are recovered as well
-
-		})
-	}
-}
-
-func removeRandShares(data [][]byte) [][]byte {
-	count := len(data)
-	// remove half of the shares randomly
-	for i := 0; i < (count / 2); {
-		ind := rand.Intn(count)
-		if len(data[ind]) == 0 {
-			continue
-		}
-		data[ind] = nil
-		i++
-	}
-	fmt.Println("removal data len", len(data))
-	return data
-}
-
 func generateRandomData(msgCount int) Data {
 	out := make([]Message, msgCount)
 	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, MsgShareSize-2) {

From d882848a132a10b2225c143879b25dd153cf8515 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:12:17 -0500
Subject: [PATCH 22/30] remove uneeded function

---
 types/shares.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/types/shares.go b/types/shares.go
index 34be98ce17..4abbf0d9bb 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -38,14 +38,6 @@ func (ns NamespacedShares) RawShares() [][]byte {
 	return res
 }
 
-func (ns NamespacedShares) NamedShares() [][]byte {
-	res := make([][]byte, len(ns))
-	for i, nsh := range ns {
-		res[i] = append(nsh.NamespaceID(), nsh.Data()...)
-	}
-	return res
-}
-
 func (tx Tx) MarshalDelimited() ([]byte, error) {
 	lenBuf := make([]byte, binary.MaxVarintLen64)
 	length := uint64(len(tx))

From b7bdfb78bd1c5d4e266584aa6d8f3e57ea86cfd1 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:12:37 -0500
Subject: [PATCH 23/30] remove comment

---
 types/shares.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/types/shares.go b/types/shares.go
index 4abbf0d9bb..93d6de8a08 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -55,9 +55,6 @@ func (m Message) MarshalDelimited() ([]byte, error) {
 	return append(lenBuf[:n], m.Data...), nil
 }
 
-// m.Data is being altered somehwere along the line, where the first two bytes are being replaces with the above append
-// the append isn't inherently wrong, it's that the data should not altered after that.
-
 // appendToShares appends raw data as shares.
 // Used for messages.
 func appendToShares(shares []NamespacedShare, id namespace.ID, rawData []byte) []NamespacedShare {

From d84fc3d7ab5495389998a3ffbe98ed14932b4859 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:17:05 -0500
Subject: [PATCH 24/30] clean up

---
 types/block.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/types/block.go b/types/block.go
index e237fb0a1a..66615214a9 100644
--- a/types/block.go
+++ b/types/block.go
@@ -217,7 +217,6 @@ func (b *Block) fillDataAvailabilityHeader() {
 
 	// record the widths
 	squareWidth := extendedDataSquare.Width()
-	fmt.Println("square width", squareWidth)
 
 	b.DataAvailabilityHeader = DataAvailabilityHeader{
 		RowsRoots:   make([]namespace.IntervalDigest, squareWidth),
@@ -265,8 +264,6 @@ func mustPush(rowTree *nmt.NamespacedMerkleTree, id namespace.ID, data []byte) {
 	}
 }
 
-// the erasured leaves are going to be longer than the non erasured leaves.
-
 // PutBlock posts and pins erasured block data to IPFS using the provided
 // ipld.NodeAdder. Note: the erasured data is currently recomputed
 func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error {

From f7085030fbf0ffa5807cb0d99227286933a62d35 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:25:38 -0500
Subject: [PATCH 25/30] fix imports

---
 types/block_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/types/block_test.go b/types/block_test.go
index 1b69028b49..d372681199 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -6,7 +6,6 @@ import (
 	stdbytes "bytes"
 	"context"
 	"encoding/hex"
-	"fmt"
 	"math"
 	"math/rand"
 	"os"
@@ -20,7 +19,6 @@ import (
 	coremock "github.com/ipfs/go-ipfs/core/mock"
 	"github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
-	"github.com/lazyledger/rsmt2d"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

From efdd4f0549b4c661949ed31c903e1c5e0e6d1470 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:27:29 -0500
Subject: [PATCH 26/30] linter gods

---
 types/shares.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/types/shares.go b/types/shares.go
index 93d6de8a08..e395ce0c1c 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -61,7 +61,7 @@ func appendToShares(shares []NamespacedShare, id namespace.ID, rawData []byte) [
 	nid := make([]byte, len(id))
 	copy(nid, id)
 	if len(rawData) <= MsgShareSize {
-		rawShare := []byte(append(nid, rawData...))
+		rawShare := append(nid, rawData...)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -85,7 +85,7 @@ func splitContiguous(id namespace.ID, rawDatas [][]byte) []NamespacedShare {
 		copy(nid, id)
 		startIndex := 0
 		rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
-		rawShare := []byte(append(append(nid, byte(startIndex)), rawData...))
+		rawShare := append(append(nid, byte(startIndex)), rawData...)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)

From 43a6bd7fd3abddd3d7e228073e7f2c77297e86f2 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Sun, 28 Mar 2021 23:30:29 -0500
Subject: [PATCH 27/30] more linter fixes

---
 types/block_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/types/block_test.go b/types/block_test.go
index d372681199..54890bc97b 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -195,8 +195,8 @@ func makeBlockIDRandom() BlockID {
 		blockHash   = make([]byte, tmhash.Size)
 		partSetHash = make([]byte, tmhash.Size)
 	)
-	rand.Read(blockHash)   //nolint: errcheck // ignore errcheck for read
-	rand.Read(partSetHash) //nolint: errcheck // ignore errcheck for read
+	rand.Read(blockHash)
+	rand.Read(partSetHash)
 	return BlockID{blockHash, PartSetHeader{123, partSetHash}}
 }
 

From 6baddea41ae240057b7150fbaaef7c2a3435c13b Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Mon, 29 Mar 2021 08:57:46 -0500
Subject: [PATCH 28/30] use appendCopy instead of manually copying

---
 types/shares.go | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/types/shares.go b/types/shares.go
index e395ce0c1c..641f92681a 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -57,11 +57,9 @@ func (m Message) MarshalDelimited() ([]byte, error) {
 
 // appendToShares appends raw data as shares.
 // Used for messages.
-func appendToShares(shares []NamespacedShare, id namespace.ID, rawData []byte) []NamespacedShare {
-	nid := make([]byte, len(id))
-	copy(nid, id)
+func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
 	if len(rawData) <= MsgShareSize {
-		rawShare := append(nid, rawData...)
+		rawShare := appendCopy(nid, rawData)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -73,7 +71,7 @@ func appendToShares(shares []NamespacedShare, id namespace.ID, rawData []byte) [
 
 // splitContiguous splits multiple raw data contiguously as shares.
 // Used for transactions, intermediate state roots, and evidence.
-func splitContiguous(id namespace.ID, rawDatas [][]byte) []NamespacedShare {
+func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
 	shares := make([]NamespacedShare, 0)
 	// Index into the outer slice of rawDatas
 	outerIndex := 0
@@ -81,11 +79,9 @@ func splitContiguous(id namespace.ID, rawDatas [][]byte) []NamespacedShare {
 	innerIndex := 0
 	for outerIndex < len(rawDatas) {
 		var rawData []byte
-		nid := make([]byte, len(id))
-		copy(nid, id)
 		startIndex := 0
 		rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
-		rawShare := append(append(nid, byte(startIndex)), rawData...)
+		rawShare := append(appendCopy(nid, []byte{byte(startIndex)}), rawData...)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -102,9 +98,7 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare {
 	rawData = rawData[MsgShareSize:]
 	for len(rawData) > 0 {
 		shareSizeOrLen := min(MsgShareSize, len(rawData))
-		rawShare := make([]byte, NamespaceSize)
-		copy(rawShare, nid)
-		rawShare = append(rawShare, rawData[:shareSizeOrLen]...)
+		rawShare := appendCopy(nid, rawData[:shareSizeOrLen])
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -174,3 +168,15 @@ func zeroPadIfNecessary(share []byte, width int) []byte {
 	}
 	return share
 }
+
+// appendCopy makes a copy of slice a and appends slices b to that copy
+func appendCopy(a []byte, b ...[]byte) []byte {
+	c := make([]byte, len(a))
+	copy(c, a)
+
+	for _, bi := range b {
+		c = append(c, bi...)
+	}
+
+	return c
+}

From 64e5bfd08d0d6f9749756bab0acef9ee99bb7a55 Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Mon, 29 Mar 2021 09:46:19 -0500
Subject: [PATCH 29/30] change name of generateRandomData to
 generateRandomMsgOnlyData

---
 types/block_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/types/block_test.go b/types/block_test.go
index 54890bc97b..a1250af095 100644
--- a/types/block_test.go
+++ b/types/block_test.go
@@ -1335,10 +1335,10 @@ func TestPutBlock(t *testing.T) {
 		expectErr bool
 		errString string
 	}{
-		{"no leaves", generateRandomData(0), false, ""},
-		{"single leaf", generateRandomData(1), false, ""},
-		{"16 leaves", generateRandomData(16), false, ""},
-		{"max square size", generateRandomData(MaxSquareSize), false, ""},
+		{"no leaves", generateRandomMsgOnlyData(0), false, ""},
+		{"single leaf", generateRandomMsgOnlyData(1), false, ""},
+		{"16 leaves", generateRandomMsgOnlyData(16), false, ""},
+		{"max square size", generateRandomMsgOnlyData(MaxSquareSize), false, ""},
 	}
 	ctx := context.Background()
 	for _, tc := range testCases {
@@ -1386,7 +1386,7 @@ func TestPutBlock(t *testing.T) {
 	}
 }
 
-func generateRandomData(msgCount int) Data {
+func generateRandomMsgOnlyData(msgCount int) Data {
 	out := make([]Message, msgCount)
 	for i, msg := range generateRandNamespacedRawData(msgCount, NamespaceSize, MsgShareSize-2) {
 		out[i] = Message{NamespaceID: msg[:NamespaceSize], Data: msg[NamespaceSize:]}

From 8bba9d9622ba0b791360f7bf633e1e4409f0127a Mon Sep 17 00:00:00 2001
From: evan-forbes <evan.samuel.forbes@gmail.com>
Date: Mon, 29 Mar 2021 13:50:38 -0500
Subject: [PATCH 30/30] use append instead of a utiltiy function

---
 types/shares.go | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/types/shares.go b/types/shares.go
index 641f92681a..649f74b774 100644
--- a/types/shares.go
+++ b/types/shares.go
@@ -59,7 +59,11 @@ func (m Message) MarshalDelimited() ([]byte, error) {
 // Used for messages.
 func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
 	if len(rawData) <= MsgShareSize {
-		rawShare := appendCopy(nid, rawData)
+		rawShare := append(append(
+			make([]byte, 0, len(nid)+len(rawData)),
+			nid...),
+			rawData...,
+		)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -81,7 +85,11 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
 		var rawData []byte
 		startIndex := 0
 		rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
-		rawShare := append(appendCopy(nid, []byte{byte(startIndex)}), rawData...)
+		rawShare := append(append(append(
+			make([]byte, 0, len(nid)+1+len(rawData)),
+			nid...),
+			byte(startIndex)),
+			rawData...)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -93,12 +101,20 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
 // shares for a particular namespace
 func split(rawData []byte, nid namespace.ID) []NamespacedShare {
 	shares := make([]NamespacedShare, 0)
-	firstRawShare := []byte(append(nid, rawData[:MsgShareSize]...))
+	firstRawShare := append(append(
+		make([]byte, 0, len(nid)+len(rawData[:MsgShareSize])),
+		nid...),
+		rawData[:MsgShareSize]...,
+	)
 	shares = append(shares, NamespacedShare{firstRawShare, nid})
 	rawData = rawData[MsgShareSize:]
 	for len(rawData) > 0 {
 		shareSizeOrLen := min(MsgShareSize, len(rawData))
-		rawShare := appendCopy(nid, rawData[:shareSizeOrLen])
+		rawShare := append(append(
+			make([]byte, 0, len(nid)+1+len(rawData[:shareSizeOrLen])),
+			nid...),
+			rawData[:shareSizeOrLen]...,
+		)
 		paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
 		share := NamespacedShare{paddedShare, nid}
 		shares = append(shares, share)
@@ -168,15 +184,3 @@ func zeroPadIfNecessary(share []byte, width int) []byte {
 	}
 	return share
 }
-
-// appendCopy makes a copy of slice a and appends slices b to that copy
-func appendCopy(a []byte, b ...[]byte) []byte {
-	c := make([]byte, len(a))
-	copy(c, a)
-
-	for _, bi := range b {
-		c = append(c, bi...)
-	}
-
-	return c
-}