From 2137d672b3388d3a022a7ce39fdd119d92bfb696 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 13:27:14 -0400 Subject: [PATCH 01/30] Export block data compute shares. --- types/block.go | 6 +++--- types/block_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/types/block.go b/types/block.go index d442ad86a9..80fb42b496 100644 --- a/types/block.go +++ b/types/block.go @@ -200,7 +200,7 @@ func (b *Block) fillHeader() { // fillDataAvailabilityHeader fills in any remaining DataAvailabilityHeader fields // that are a function of the block data. func (b *Block) fillDataAvailabilityHeader() { - namespacedShares := b.Data.computeShares() + namespacedShares := b.Data.ComputeShares() shares := namespacedShares.RawShares() if len(shares) == 0 { // no shares -> no row/colum roots -> hash(empty) @@ -272,7 +272,7 @@ func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error } // recompute the shares - namespacedShares := b.Data.computeShares() + namespacedShares := b.Data.ComputeShares() shares := namespacedShares.RawShares() // don't do anything if there is no data to put on IPFS @@ -1336,7 +1336,7 @@ func (msgs Messages) splitIntoShares(shareSize int) NamespacedShares { return shares } -func (data *Data) computeShares() NamespacedShares { +func (data *Data) ComputeShares() NamespacedShares { // TODO(ismail): splitting into shares should depend on the block size and layout // see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/block_proposer.md#laying-out-transactions-and-messages diff --git a/types/block_test.go b/types/block_test.go index 87f42479b3..23b69a7054 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -1360,7 +1360,7 @@ func TestPutBlock(t *testing.T) { defer cancel() block.fillDataAvailabilityHeader() - tc.blockData.computeShares() + tc.blockData.ComputeShares() for _, rowRoot := range block.DataAvailabilityHeader.RowsRoots.Bytes() { // recreate the cids using only the computed roots cid, err := nodes.CidFromNamespacedSha256(rowRoot) From be307d5ef84e0527707765a1d91bf5ba53b838b3 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 13:28:04 -0400 Subject: [PATCH 02/30] Add doc on exported func. --- types/block.go | 1 + 1 file changed, 1 insertion(+) diff --git a/types/block.go b/types/block.go index 80fb42b496..1725a7febd 100644 --- a/types/block.go +++ b/types/block.go @@ -1336,6 +1336,7 @@ func (msgs Messages) splitIntoShares(shareSize int) NamespacedShares { return shares } +// ComputeShares splits block data into shares of an original data square. func (data *Data) ComputeShares() NamespacedShares { // TODO(ismail): splitting into shares should depend on the block size and layout // see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/block_proposer.md#laying-out-transactions-and-messages From e2ba998655f080c497fb25a5322cb84a88b73948 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 16:48:24 -0400 Subject: [PATCH 03/30] Refactor to use ShareSize constant directly. --- types/block.go | 20 ++++++++++---------- types/shares_test.go | 17 ++++++----------- types/tx.go | 4 ++-- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/types/block.go b/types/block.go index 1725a7febd..70c865dc1e 100644 --- a/types/block.go +++ b/types/block.go @@ -1312,26 +1312,26 @@ type IntermediateStateRoots struct { RawRootsList []tmbytes.HexBytes `json:"intermediate_roots"` } -func (roots IntermediateStateRoots) splitIntoShares(shareSize int) NamespacedShares { +func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) for _, root := range roots.RawRootsList { rawData, err := root.MarshalDelimited() if err != nil { panic(fmt.Sprintf("app returned intermediate state root that can not be encoded %#v", root)) } - shares = appendToShares(shares, IntermediateStateRootsNamespaceID, rawData, shareSize) + shares = appendToShares(shares, IntermediateStateRootsNamespaceID, rawData, ShareSize) } return shares } -func (msgs Messages) splitIntoShares(shareSize int) NamespacedShares { +func (msgs Messages) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) for _, m := range msgs.MessagesList { rawData, err := m.MarshalDelimited() if err != nil { panic(fmt.Sprintf("app accepted a Message that can not be encoded %#v", m)) } - shares = appendToShares(shares, m.NamespaceID, rawData, shareSize) + shares = appendToShares(shares, m.NamespaceID, rawData, ShareSize) } return shares } @@ -1342,12 +1342,12 @@ func (data *Data) ComputeShares() NamespacedShares { // see: https://github.com/lazyledger/lazyledger-specs/blob/master/specs/block_proposer.md#laying-out-transactions-and-messages // reserved shares: - txShares := data.Txs.splitIntoShares(ShareSize) - intermRootsShares := data.IntermediateStateRoots.splitIntoShares(ShareSize) - evidenceShares := data.Evidence.splitIntoShares(ShareSize) + txShares := data.Txs.splitIntoShares() + intermRootsShares := data.IntermediateStateRoots.splitIntoShares() + evidenceShares := data.Evidence.splitIntoShares() // application data shares from messages: - msgShares := data.Messages.splitIntoShares(ShareSize) + msgShares := data.Messages.splitIntoShares() curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares) // FIXME(ismail): this is not a power of two @@ -1588,7 +1588,7 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error { return nil } -func (data *EvidenceData) splitIntoShares(shareSize int) NamespacedShares { +func (data *EvidenceData) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) for _, ev := range data.Evidence { var rawData []byte @@ -1609,7 +1609,7 @@ func (data *EvidenceData) splitIntoShares(shareSize int) NamespacedShares { if err != nil { panic(fmt.Sprintf("evidence included in evidence pool that can not be encoded %#v, err: %v", ev, err)) } - shares = appendToShares(shares, EvidenceNamespaceID, rawData, shareSize) + shares = appendToShares(shares, EvidenceNamespaceID, rawData, ShareSize) } return shares } diff --git a/types/shares_test.go b/types/shares_test.go index 4be02f9f98..a455c86170 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -10,7 +10,7 @@ import ( ) type splitter interface { - splitIntoShares(shareSize int) NamespacedShares + splitIntoShares() NamespacedShares } func TestMakeShares(t *testing.T) { @@ -41,8 +41,7 @@ func TestMakeShares(t *testing.T) { } type args struct { - data splitter - shareSize int + data splitter } tests := []struct { name string @@ -54,7 +53,6 @@ func TestMakeShares(t *testing.T) { data: &EvidenceData{ Evidence: []Evidence{testEvidence}, }, - shareSize: ShareSize, }, NamespacedShares{NamespacedShare{ Share: testEvidenceBytes[:ShareSize], ID: reservedEvidenceNamespaceID, @@ -65,8 +63,7 @@ func TestMakeShares(t *testing.T) { }, {"small LL Tx", args{ - data: Txs{smolTx}, - shareSize: ShareSize, + data: Txs{smolTx}, }, NamespacedShares{ NamespacedShare{ @@ -77,8 +74,7 @@ func TestMakeShares(t *testing.T) { }, {"one large LL Tx", args{ - data: Txs{largeTx}, - shareSize: ShareSize, + data: Txs{largeTx}, }, NamespacedShares{ NamespacedShare{ @@ -93,8 +89,7 @@ func TestMakeShares(t *testing.T) { }, {"ll-app message", args{ - data: Messages{[]Message{msg1}}, - shareSize: ShareSize, + data: Messages{[]Message{msg1}}, }, NamespacedShares{ NamespacedShare{zeroPadIfNecessary(msg1Marshaled, ShareSize), msg1.NamespaceID}, @@ -105,7 +100,7 @@ func TestMakeShares(t *testing.T) { tt := tt // stupid scopelint :-/ i := i t.Run(tt.name, func(t *testing.T) { - if got := tt.args.data.splitIntoShares(tt.args.shareSize); !reflect.DeepEqual(got, tt.want) { + if got := tt.args.data.splitIntoShares(); !reflect.DeepEqual(got, tt.want) { t.Errorf("%v: makeShares() = \n%v\nwant\n%v", i, got, tt.want) } }) diff --git a/types/tx.go b/types/tx.go index d011cce4e1..d63785a259 100644 --- a/types/tx.go +++ b/types/tx.go @@ -79,14 +79,14 @@ func (txs Txs) Proof(i int) TxProof { } } -func (txs Txs) splitIntoShares(shareSize int) NamespacedShares { +func (txs Txs) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) for _, tx := range txs { rawData, err := tx.MarshalDelimited() if err != nil { panic(fmt.Sprintf("included Tx in mem-pool that can not be encoded %v", tx)) } - shares = appendToShares(shares, TxNamespaceID, rawData, shareSize) + shares = appendToShares(shares, TxNamespaceID, rawData, ShareSize) } return shares } From b16f200333b3f4d9732538dad413c2dfabd4fed2 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 19:51:45 -0400 Subject: [PATCH 04/30] Change message splitting to prefix namespace ID. --- types/block.go | 10 +++++++--- types/consts.go | 3 +++ types/shares.go | 45 ++++++++++++++++++++++++++++++++++----------- types/tx.go | 4 +++- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/types/block.go b/types/block.go index 70c865dc1e..39c23ac9b9 100644 --- a/types/block.go +++ b/types/block.go @@ -1314,13 +1314,15 @@ type IntermediateStateRoots struct { func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) + rawDatas := make([][]byte, len(roots.RawRootsList)) for _, root := range roots.RawRootsList { rawData, err := root.MarshalDelimited() if err != nil { panic(fmt.Sprintf("app returned intermediate state root that can not be encoded %#v", root)) } - shares = appendToShares(shares, IntermediateStateRootsNamespaceID, rawData, ShareSize) + rawDatas = append(rawDatas, rawData) } + shares = appendToSharesContiguous(shares, IntermediateStateRootsNamespaceID, rawDatas) return shares } @@ -1331,7 +1333,7 @@ func (msgs Messages) splitIntoShares() NamespacedShares { if err != nil { panic(fmt.Sprintf("app accepted a Message that can not be encoded %#v", m)) } - shares = appendToShares(shares, m.NamespaceID, rawData, ShareSize) + shares = appendToShares(shares, m.NamespaceID, rawData) } return shares } @@ -1590,6 +1592,7 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error { func (data *EvidenceData) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) + rawDatas := make([][]byte, len(data.Evidence)) for _, ev := range data.Evidence { var rawData []byte var err error @@ -1609,8 +1612,9 @@ func (data *EvidenceData) splitIntoShares() NamespacedShares { if err != nil { panic(fmt.Sprintf("evidence included in evidence pool that can not be encoded %#v, err: %v", ev, err)) } - shares = appendToShares(shares, EvidenceNamespaceID, rawData, ShareSize) + rawDatas = append(rawDatas, rawData) } + shares = appendToSharesContiguous(shares, EvidenceNamespaceID, rawDatas) return shares } diff --git a/types/consts.go b/types/consts.go index 78872f9a1a..e433f9048e 100644 --- a/types/consts.go +++ b/types/consts.go @@ -16,6 +16,9 @@ const ( // NamespaceSize is the namespace size in bytes. NamespaceSize = 8 + // ShareReservedBytes is the reserved bytes for contiguous appends. + ShareReservedBytes = 1 + // MaxSquareSize is the maximum number of // rows/columns of the original data shares in square layout. // Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec. diff --git a/types/shares.go b/types/shares.go index 71f41a1f9d..0a3d675032 100644 --- a/types/shares.go +++ b/types/shares.go @@ -56,28 +56,51 @@ func (m Message) MarshalDelimited() ([]byte, error) { return append(lenBuf[:n], m.Data...), nil } -func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte, shareSize int) []NamespacedShare { - if len(rawData) < shareSize { - rawShare := rawData - paddedShare := zeroPadIfNecessary(rawShare, shareSize) +// appendToSharesContiguous appends one raw data separately as shares +// Used for messages +func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { + adjustedSize := ShareSize - NamespaceSize + if len(rawData) < adjustedSize { + rawShare := []byte(append(nid, rawData...)) + paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} shares = append(shares, share) - } else { // len(rawData) >= shareSize - shares = append(shares, split(rawData, shareSize, nid)...) + } else { // len(rawData) >= adjustedSize + shares = append(shares, split(rawData, nid)...) + } + return shares +} + +// appendToSharesContiguous appends multiple raw data contiguously as shares +// Used for transactions, intermediate state roots, and evidence +func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDatas [][]byte) []NamespacedShare { + adjustedSize := ShareSize - NamespaceSize - ShareReservedBytes + for _, rawData := range rawDatas { + if len(rawData) < adjustedSize { + rawShare := []byte(append(append(nid, byte(0)), rawData...)) + paddedShare := zeroPadIfNecessary(rawShare, ShareSize) + share := NamespacedShare{paddedShare, nid} + shares = append(shares, share) + } else { // len(rawData) >= adjustedSize + // TODO(john) this needs to be splitContiguous, etc. + shares = append(shares, split(rawData, nid)...) + } } return shares } // TODO(ismail): implement corresponding merge method for clients requesting // shares for a particular namespace -func split(rawData []byte, shareSize int, nid namespace.ID) []NamespacedShare { +func split(rawData []byte, nid namespace.ID) []NamespacedShare { + adjustedSize := ShareSize - NamespaceSize shares := make([]NamespacedShare, 0) - firstRawShare := rawData[:shareSize] + firstRawShare := []byte(append(nid, rawData[:adjustedSize]...)) shares = append(shares, NamespacedShare{firstRawShare, nid}) - rawData = rawData[shareSize:] + rawData = rawData[adjustedSize:] for len(rawData) > 0 { - shareSizeOrLen := min(shareSize, len(rawData)) - paddedShare := zeroPadIfNecessary(rawData[:shareSizeOrLen], shareSize) + shareSizeOrLen := min(adjustedSize, len(rawData)) + rawShare := []byte(append(nid, rawData[:shareSizeOrLen]...)) + paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} shares = append(shares, share) rawData = rawData[shareSizeOrLen:] diff --git a/types/tx.go b/types/tx.go index d63785a259..257288e8b6 100644 --- a/types/tx.go +++ b/types/tx.go @@ -81,13 +81,15 @@ func (txs Txs) Proof(i int) TxProof { func (txs Txs) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) + rawDatas := make([][]byte, len(txs)) for _, tx := range txs { rawData, err := tx.MarshalDelimited() if err != nil { panic(fmt.Sprintf("included Tx in mem-pool that can not be encoded %v", tx)) } - shares = appendToShares(shares, TxNamespaceID, rawData, ShareSize) + rawDatas = append(rawDatas, rawData) } + shares = appendToSharesContiguous(shares, TxNamespaceID, rawDatas) return shares } From 7d23abf3c6d4280a7e96b3c1d0ae68dcf56789c3 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 20:12:50 -0400 Subject: [PATCH 05/30] Get ready for chunking contiguous. --- types/shares.go | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/types/shares.go b/types/shares.go index 0a3d675032..675e92f22f 100644 --- a/types/shares.go +++ b/types/shares.go @@ -59,7 +59,7 @@ func (m Message) MarshalDelimited() ([]byte, error) { // appendToSharesContiguous appends one raw data separately as shares // Used for messages func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { - adjustedSize := ShareSize - NamespaceSize + const adjustedSize = ShareSize - NamespaceSize if len(rawData) < adjustedSize { rawShare := []byte(append(nid, rawData...)) paddedShare := zeroPadIfNecessary(rawShare, ShareSize) @@ -74,17 +74,19 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) // appendToSharesContiguous appends multiple raw data contiguously as shares // Used for transactions, intermediate state roots, and evidence func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDatas [][]byte) []NamespacedShare { - adjustedSize := ShareSize - NamespaceSize - ShareReservedBytes - for _, rawData := range rawDatas { - if len(rawData) < adjustedSize { - rawShare := []byte(append(append(nid, byte(0)), rawData...)) - paddedShare := zeroPadIfNecessary(rawShare, ShareSize) - share := NamespacedShare{paddedShare, nid} - shares = append(shares, share) - } else { // len(rawData) >= adjustedSize - // TODO(john) this needs to be splitContiguous, etc. - shares = append(shares, split(rawData, nid)...) - } + const adjustedSize = ShareSize - NamespaceSize - ShareReservedBytes + // Index into the outer slice of rawDatas + outerIndex := 0 + // Index into the inner slice of rawDatas + innerIndex := 0 + for outerIndex < len(rawDatas) { + rawData := make([]byte, adjustedSize) + startIndex := 0 + rawData, outerIndex, innerIndex, startIndex = getNextBytes(rawDatas, outerIndex, innerIndex, adjustedSize) + rawShare := []byte(append(append(nid, byte(startIndex)), rawData...)) + paddedShare := zeroPadIfNecessary(rawShare, ShareSize) + share := NamespacedShare{paddedShare, nid} + shares = append(shares, share) } return shares } @@ -92,7 +94,7 @@ func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDat // TODO(ismail): implement corresponding merge method for clients requesting // shares for a particular namespace func split(rawData []byte, nid namespace.ID) []NamespacedShare { - adjustedSize := ShareSize - NamespaceSize + const adjustedSize = ShareSize - NamespaceSize shares := make([]NamespacedShare, 0) firstRawShare := []byte(append(nid, rawData[:adjustedSize]...)) shares = append(shares, NamespacedShare{firstRawShare, nid}) @@ -108,6 +110,13 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare { return shares } +func getNextBytes(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { + rawData := make([]byte, width) + startIndex := 0 + // TODO do this + return rawData, outerIndex, innerIndex, startIndex +} + func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares { shares := make([]NamespacedShare, n) for i := 0; i < n; i++ { From 2331a726a7ce5057349497860e9603e92f59c137 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 20:29:34 -0400 Subject: [PATCH 06/30] Add start index comp. --- types/shares.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/types/shares.go b/types/shares.go index 675e92f22f..d1a5b3453b 100644 --- a/types/shares.go +++ b/types/shares.go @@ -82,7 +82,7 @@ func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDat for outerIndex < len(rawDatas) { rawData := make([]byte, adjustedSize) startIndex := 0 - rawData, outerIndex, innerIndex, startIndex = getNextBytes(rawDatas, outerIndex, innerIndex, adjustedSize) + rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, adjustedSize) rawShare := []byte(append(append(nid, byte(startIndex)), rawData...)) paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} @@ -110,9 +110,18 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare { return shares } -func getNextBytes(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { +func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { rawData := make([]byte, width) - startIndex := 0 + startIndex := len(rawDatas[outerIndex]) - innerIndex - 1 + // If the start index would go past the end of the share, no transaction begins in this share + if startIndex >= width { + startIndex = 0 + } + // Offset by the fixed reserved bytes at the beginning of the share + if startIndex > 0 { + startIndex += NamespaceSize + ShareReservedBytes + } + // TODO do this return rawData, outerIndex, innerIndex, startIndex } From 665b611a919191462f5f1655fb4686aa680005cb Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 20:47:35 -0400 Subject: [PATCH 07/30] Implement chunking for contiguous. --- types/shares.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/types/shares.go b/types/shares.go index d1a5b3453b..9483932985 100644 --- a/types/shares.go +++ b/types/shares.go @@ -111,7 +111,7 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare { } func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { - rawData := make([]byte, width) + rawData := make([]byte, 0, width) startIndex := len(rawDatas[outerIndex]) - innerIndex - 1 // If the start index would go past the end of the share, no transaction begins in this share if startIndex >= width { @@ -122,7 +122,18 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) startIndex += NamespaceSize + ShareReservedBytes } - // TODO do this + curIndex := 0 + for curIndex < width && outerIndex < len(rawDatas) { + bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex-1, width-curIndex-1) + rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...) + innerIndex += bytesToFetch + if innerIndex >= len(rawDatas[outerIndex]) { + innerIndex = 0 + outerIndex++ + } + curIndex += bytesToFetch + } + return rawData, outerIndex, innerIndex, startIndex } From 5a0c5eaf604f4709773ea085b23cbc28e68c07a4 Mon Sep 17 00:00:00 2001 From: John Adler Date: Mon, 22 Mar 2021 20:54:49 -0400 Subject: [PATCH 08/30] Make tests print newline. --- types/shares_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index a455c86170..7cfee70a7b 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -37,7 +37,7 @@ func TestMakeShares(t *testing.T) { } msg1Marshaled, _ := msg1.MarshalDelimited() if err != nil { - t.Fatalf("Could not encode evidence: %v, error: %v", testEvidence, err) + t.Fatalf("Could not encode evidence: %v, error: %v\n", testEvidence, err) } type args struct { @@ -101,7 +101,7 @@ func TestMakeShares(t *testing.T) { i := i t.Run(tt.name, func(t *testing.T) { if got := tt.args.data.splitIntoShares(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("%v: makeShares() = \n%v\nwant\n%v", i, got, tt.want) + t.Errorf("%v: makeShares() = \n%v\nwant\n%v\n", i, got, tt.want) } }) } From 95f2fb737e2c89c52ad356036167ffff1024b99b Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 08:07:19 -0400 Subject: [PATCH 09/30] Remove superfluous init. --- types/shares.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/shares.go b/types/shares.go index 9483932985..7ceb405437 100644 --- a/types/shares.go +++ b/types/shares.go @@ -80,7 +80,7 @@ func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDat // Index into the inner slice of rawDatas innerIndex := 0 for outerIndex < len(rawDatas) { - rawData := make([]byte, adjustedSize) + var rawData []byte startIndex := 0 rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, adjustedSize) rawShare := []byte(append(append(nid, byte(startIndex)), rawData...)) From 2a68a845e5ece7254168b4f50a9124a1945e1c77 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 08:55:10 -0400 Subject: [PATCH 10/30] Update types/block.go Fix to only reserve slice, not allocate. Co-authored-by: Ismail Khoffi --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index 39c23ac9b9..09dea45a90 100644 --- a/types/block.go +++ b/types/block.go @@ -1592,7 +1592,7 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error { func (data *EvidenceData) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) - rawDatas := make([][]byte, len(data.Evidence)) + rawDatas := make([][]byte, 0, len(data.Evidence)) for _, ev := range data.Evidence { var rawData []byte var err error From c956a638ae42469d23a7295c8e13af0efba64e42 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 08:55:32 -0400 Subject: [PATCH 11/30] Fix to reserve, not allocate. --- types/block.go | 2 +- types/tx.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/types/block.go b/types/block.go index 39c23ac9b9..5e5723bae6 100644 --- a/types/block.go +++ b/types/block.go @@ -1314,7 +1314,7 @@ type IntermediateStateRoots struct { func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) - rawDatas := make([][]byte, len(roots.RawRootsList)) + rawDatas := make([][]byte, 0, len(roots.RawRootsList)) for _, root := range roots.RawRootsList { rawData, err := root.MarshalDelimited() if err != nil { diff --git a/types/tx.go b/types/tx.go index 257288e8b6..555e4d304d 100644 --- a/types/tx.go +++ b/types/tx.go @@ -81,7 +81,7 @@ func (txs Txs) Proof(i int) TxProof { func (txs Txs) splitIntoShares() NamespacedShares { shares := make([]NamespacedShare, 0) - rawDatas := make([][]byte, len(txs)) + rawDatas := make([][]byte, 0, len(txs)) for _, tx := range txs { rawData, err := tx.MarshalDelimited() if err != nil { From fe2c64649aa836073046744ffdda48699d2a1bd3 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 08:59:16 -0400 Subject: [PATCH 12/30] Clean up. --- types/shares_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/types/shares_test.go b/types/shares_test.go index 7cfee70a7b..132114d771 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -16,7 +16,6 @@ type splitter interface { func TestMakeShares(t *testing.T) { reservedTxNamespaceID := append(bytes.Repeat([]byte{0}, 7), 1) reservedEvidenceNamespaceID := append(bytes.Repeat([]byte{0}, 7), 3) - // resveredIntermediateStateRootsNamespaceID := append(bytes.Repeat([]byte{0}, 7), 2) val := NewMockPV() blockID := makeBlockID([]byte("blockhash"), 1000, []byte("partshash")) blockID2 := makeBlockID([]byte("blockhash2"), 1000, []byte("partshash")) From 0a47c65d81311ef089e640ea687cf1e80a317084 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 09:08:12 -0400 Subject: [PATCH 13/30] Add termination condition. --- types/shares.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/types/shares.go b/types/shares.go index 7ceb405437..5a03271631 100644 --- a/types/shares.go +++ b/types/shares.go @@ -125,6 +125,10 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex := 0 for curIndex < width && outerIndex < len(rawDatas) { bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex-1, width-curIndex-1) + if bytesToFetch == 0 { + innerIndex = 0 + outerIndex++ + } rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...) innerIndex += bytesToFetch if innerIndex >= len(rawDatas[outerIndex]) { From 488ae0ed5e380c24d646c870dc4ad6441eb92205 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 09:08:56 -0400 Subject: [PATCH 14/30] Fix continue. --- types/shares.go | 1 + 1 file changed, 1 insertion(+) diff --git a/types/shares.go b/types/shares.go index 5a03271631..dc0511c357 100644 --- a/types/shares.go +++ b/types/shares.go @@ -128,6 +128,7 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) if bytesToFetch == 0 { innerIndex = 0 outerIndex++ + continue } rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...) innerIndex += bytesToFetch From e0c09404f33830f85d40b6c3de3c06bf8cc43c3a Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 09:12:54 -0400 Subject: [PATCH 15/30] Rename append contiguous to split contiguous. --- types/block.go | 6 ++---- types/shares.go | 5 +++-- types/tx.go | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/types/block.go b/types/block.go index ce433d7e2e..66615214a9 100644 --- a/types/block.go +++ b/types/block.go @@ -1313,7 +1313,6 @@ type IntermediateStateRoots struct { } func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { - shares := make([]NamespacedShare, 0) rawDatas := make([][]byte, 0, len(roots.RawRootsList)) for _, root := range roots.RawRootsList { rawData, err := root.MarshalDelimited() @@ -1322,7 +1321,7 @@ func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares { } rawDatas = append(rawDatas, rawData) } - shares = appendToSharesContiguous(shares, IntermediateStateRootsNamespaceID, rawDatas) + shares := splitContiguous(IntermediateStateRootsNamespaceID, rawDatas) return shares } @@ -1591,7 +1590,6 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error { } func (data *EvidenceData) splitIntoShares() NamespacedShares { - shares := make([]NamespacedShare, 0) rawDatas := make([][]byte, 0, len(data.Evidence)) for _, ev := range data.Evidence { var rawData []byte @@ -1614,7 +1612,7 @@ func (data *EvidenceData) splitIntoShares() NamespacedShares { } rawDatas = append(rawDatas, rawData) } - shares = appendToSharesContiguous(shares, EvidenceNamespaceID, rawDatas) + shares := splitContiguous(EvidenceNamespaceID, rawDatas) return shares } diff --git a/types/shares.go b/types/shares.go index dc0511c357..4703759ec9 100644 --- a/types/shares.go +++ b/types/shares.go @@ -71,9 +71,10 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) return shares } -// appendToSharesContiguous appends multiple raw data contiguously as shares +// splitContiguous splits multiple raw data contiguously as shares // Used for transactions, intermediate state roots, and evidence -func appendToSharesContiguous(shares []NamespacedShare, nid namespace.ID, rawDatas [][]byte) []NamespacedShare { +func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { + shares := make([]NamespacedShare, 0) const adjustedSize = ShareSize - NamespaceSize - ShareReservedBytes // Index into the outer slice of rawDatas outerIndex := 0 diff --git a/types/tx.go b/types/tx.go index 555e4d304d..fe68609ada 100644 --- a/types/tx.go +++ b/types/tx.go @@ -80,7 +80,6 @@ func (txs Txs) Proof(i int) TxProof { } func (txs Txs) splitIntoShares() NamespacedShares { - shares := make([]NamespacedShare, 0) rawDatas := make([][]byte, 0, len(txs)) for _, tx := range txs { rawData, err := tx.MarshalDelimited() @@ -89,7 +88,7 @@ func (txs Txs) splitIntoShares() NamespacedShares { } rawDatas = append(rawDatas, rawData) } - shares = appendToSharesContiguous(shares, TxNamespaceID, rawDatas) + shares := splitContiguous(TxNamespaceID, rawDatas) return shares } From 8e2e36db94979ae47e3c2b8574a73ff3dda1e6f8 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 15:49:01 -0400 Subject: [PATCH 16/30] Fix bytes to fetch computation. --- types/shares.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/shares.go b/types/shares.go index 4703759ec9..50956a902c 100644 --- a/types/shares.go +++ b/types/shares.go @@ -125,7 +125,7 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex := 0 for curIndex < width && outerIndex < len(rawDatas) { - bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex-1, width-curIndex-1) + bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex) if bytesToFetch == 0 { innerIndex = 0 outerIndex++ From 99fb0e26d42fbfd41b7991aea1af861ebc520bee Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 15:52:50 -0400 Subject: [PATCH 17/30] Fix start index computation. --- types/shares.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/types/shares.go b/types/shares.go index 50956a902c..40cbf19098 100644 --- a/types/shares.go +++ b/types/shares.go @@ -140,6 +140,21 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex += bytesToFetch } + // Check if there is any more data, if not then we've reached the end + // and start index of next is 0 + for outerIndex < len(rawDatas) { + bytesToFetch := len(rawDatas[outerIndex]) - innerIndex + if bytesToFetch == 0 { + innerIndex = 0 + outerIndex++ + continue + } + break + } + if outerIndex >= len(rawDatas) { + startIndex = 0 + } + return rawData, outerIndex, innerIndex, startIndex } From d00a9bc8dd0ac5b99edc76ec5ac5d8474115a82b Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 15:53:05 -0400 Subject: [PATCH 18/30] Update test for small tx. --- types/shares_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/shares_test.go b/types/shares_test.go index 132114d771..daca5356c0 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -66,7 +66,7 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: zeroPadIfNecessary(smolTxLenDelimited, ShareSize), + Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(smolTxLenDelimited, ShareSize-NamespaceSize-ShareReservedBytes)...), ID: reservedTxNamespaceID, }, }, From 132523ac646512feb1e53d62773805e758ba18e0 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 15:57:27 -0400 Subject: [PATCH 19/30] Fix test for message. --- types/shares_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/types/shares_test.go b/types/shares_test.go index daca5356c0..1f7a6b31fd 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -91,7 +91,10 @@ func TestMakeShares(t *testing.T) { data: Messages{[]Message{msg1}}, }, NamespacedShares{ - NamespacedShare{zeroPadIfNecessary(msg1Marshaled, ShareSize), msg1.NamespaceID}, + NamespacedShare{ + append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, ShareSize-NamespaceSize)...), + msg1.NamespaceID, + }, }, }, } From d9adcfd699b62ecf064f74c123a978ee74a4401f Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 15:57:55 -0400 Subject: [PATCH 20/30] Clean up. --- types/shares_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index 1f7a6b31fd..691199253b 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -92,8 +92,8 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, ShareSize-NamespaceSize)...), - msg1.NamespaceID, + Share: append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, ShareSize-NamespaceSize)...), + ID: msg1.NamespaceID, }, }, }, From b6c6e300ee4e84de9a1009e05abd7d353f56b1fa Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 16:59:06 -0400 Subject: [PATCH 21/30] Fix evidence test. --- types/shares_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index 691199253b..bc22ca1273 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -38,6 +38,8 @@ func TestMakeShares(t *testing.T) { if err != nil { t.Fatalf("Could not encode evidence: %v, error: %v\n", testEvidence, err) } + txShareSize := ShareSize - NamespaceSize - ShareReservedBytes + msgShareSize := ShareSize - NamespaceSize type args struct { data splitter @@ -53,10 +55,10 @@ func TestMakeShares(t *testing.T) { Evidence: []Evidence{testEvidence}, }, }, NamespacedShares{NamespacedShare{ - Share: testEvidenceBytes[:ShareSize], + Share: append(append(reservedEvidenceNamespaceID, byte(0)), testEvidenceBytes[:txShareSize]...), ID: reservedEvidenceNamespaceID, }, NamespacedShare{ - Share: zeroPadIfNecessary(testEvidenceBytes[ShareSize:], ShareSize), + Share: append(append(reservedEvidenceNamespaceID, byte(0)), zeroPadIfNecessary(testEvidenceBytes[txShareSize:], txShareSize)...), ID: reservedEvidenceNamespaceID, }}, }, @@ -66,7 +68,7 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(smolTxLenDelimited, ShareSize-NamespaceSize-ShareReservedBytes)...), + Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(smolTxLenDelimited, txShareSize)...), ID: reservedTxNamespaceID, }, }, @@ -92,7 +94,7 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, ShareSize-NamespaceSize)...), + Share: append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, msgShareSize)...), ID: msg1.NamespaceID, }, }, From d76c8386b10479a289fa6b962fdca539b4564b98 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 17:00:48 -0400 Subject: [PATCH 22/30] Fix large tx test. --- types/shares_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index bc22ca1273..832f18aa53 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -79,11 +79,11 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: Share(largeTxLenDelimited[:ShareSize]), + Share: append(append(reservedTxNamespaceID, byte(0)), largeTxLenDelimited[:txShareSize]...), ID: reservedTxNamespaceID, }, NamespacedShare{ - Share: zeroPadIfNecessary(largeTxLenDelimited[ShareSize:], ShareSize), + Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(largeTxLenDelimited[txShareSize:], txShareSize)...), ID: reservedTxNamespaceID, }, }, From 8f25a9b91033a7f1a3638c9dd2e953fb25c38e3f Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 17:10:04 -0400 Subject: [PATCH 23/30] Fix lint. --- types/shares_test.go | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index 832f18aa53..271d93a3a6 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -55,11 +55,17 @@ func TestMakeShares(t *testing.T) { Evidence: []Evidence{testEvidence}, }, }, NamespacedShares{NamespacedShare{ - Share: append(append(reservedEvidenceNamespaceID, byte(0)), testEvidenceBytes[:txShareSize]...), - ID: reservedEvidenceNamespaceID, + Share: append( + append(reservedEvidenceNamespaceID, byte(0)), + testEvidenceBytes[:txShareSize]..., + ), + ID: reservedEvidenceNamespaceID, }, NamespacedShare{ - Share: append(append(reservedEvidenceNamespaceID, byte(0)), zeroPadIfNecessary(testEvidenceBytes[txShareSize:], txShareSize)...), - ID: reservedEvidenceNamespaceID, + Share: append( + append(reservedEvidenceNamespaceID, byte(0)), + zeroPadIfNecessary(testEvidenceBytes[txShareSize:], txShareSize)..., + ), + ID: reservedEvidenceNamespaceID, }}, }, {"small LL Tx", @@ -68,8 +74,11 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(smolTxLenDelimited, txShareSize)...), - ID: reservedTxNamespaceID, + Share: append( + append(reservedTxNamespaceID, byte(0)), + zeroPadIfNecessary(smolTxLenDelimited, txShareSize)..., + ), + ID: reservedTxNamespaceID, }, }, }, @@ -79,12 +88,18 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: append(append(reservedTxNamespaceID, byte(0)), largeTxLenDelimited[:txShareSize]...), - ID: reservedTxNamespaceID, + Share: append( + append(reservedTxNamespaceID, byte(0)), + largeTxLenDelimited[:txShareSize]..., + ), + ID: reservedTxNamespaceID, }, NamespacedShare{ - Share: append(append(reservedTxNamespaceID, byte(0)), zeroPadIfNecessary(largeTxLenDelimited[txShareSize:], txShareSize)...), - ID: reservedTxNamespaceID, + Share: append( + append(reservedTxNamespaceID, byte(0)), + zeroPadIfNecessary(largeTxLenDelimited[txShareSize:], txShareSize)..., + ), + ID: reservedTxNamespaceID, }, }, }, @@ -94,8 +109,11 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{ NamespacedShare{ - Share: append([]byte(msg1.NamespaceID), zeroPadIfNecessary(msg1Marshaled, msgShareSize)...), - ID: msg1.NamespaceID, + Share: append( + []byte(msg1.NamespaceID), + zeroPadIfNecessary(msg1Marshaled, msgShareSize)..., + ), + ID: msg1.NamespaceID, }, }, }, From defcd50988c99dfc1cca3051d3ee05cbbcf9f7d8 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 20:43:56 -0400 Subject: [PATCH 24/30] Add test for two contiguous. --- types/shares_test.go | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/types/shares_test.go b/types/shares_test.go index 271d93a3a6..b755a15ae7 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -38,8 +38,8 @@ func TestMakeShares(t *testing.T) { if err != nil { t.Fatalf("Could not encode evidence: %v, error: %v\n", testEvidence, err) } - txShareSize := ShareSize - NamespaceSize - ShareReservedBytes - msgShareSize := ShareSize - NamespaceSize + const txShareSize = ShareSize - NamespaceSize - ShareReservedBytes + const msgShareSize = ShareSize - NamespaceSize type args struct { data splitter @@ -103,6 +103,30 @@ func TestMakeShares(t *testing.T) { }, }, }, + {"large then small LL Tx", + args{ + data: Txs{largeTx, smolTx}, + }, + NamespacedShares{ + NamespacedShare{ + Share: append( + append(reservedTxNamespaceID, byte(0)), + largeTxLenDelimited[:txShareSize]..., + ), + ID: reservedTxNamespaceID, + }, + NamespacedShare{ + Share: append( + append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-txShareSize+NamespaceSize+ShareReservedBytes)), + zeroPadIfNecessary( + append(largeTxLenDelimited[txShareSize:], smolTxLenDelimited...), + txShareSize, + )..., + ), + ID: reservedTxNamespaceID, + }, + }, + }, {"ll-app message", args{ data: Messages{[]Message{msg1}}, @@ -122,8 +146,9 @@ func TestMakeShares(t *testing.T) { tt := tt // stupid scopelint :-/ i := i t.Run(tt.name, func(t *testing.T) { - if got := tt.args.data.splitIntoShares(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("%v: makeShares() = \n%v\nwant\n%v\n", i, got, tt.want) + got := tt.args.data.splitIntoShares() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("%v: makeShares() = \n%+v\nwant\n%+v\n", i, got, tt.want) } }) } From 25f5d94d6cd15bfc014df5a3d713f950ec61ea94 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 23 Mar 2021 21:00:11 -0400 Subject: [PATCH 25/30] Fix start index computation. --- types/shares.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/types/shares.go b/types/shares.go index 40cbf19098..9a677c2568 100644 --- a/types/shares.go +++ b/types/shares.go @@ -113,24 +113,27 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare { func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { rawData := make([]byte, 0, width) - startIndex := len(rawDatas[outerIndex]) - innerIndex - 1 - // If the start index would go past the end of the share, no transaction begins in this share - if startIndex >= width { - startIndex = 0 - } - // Offset by the fixed reserved bytes at the beginning of the share - if startIndex > 0 { - startIndex += NamespaceSize + ShareReservedBytes - } + startIndex := 0 + firstBytesToFetch := 0 curIndex := 0 for curIndex < width && outerIndex < len(rawDatas) { bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex) + // Prune any extra 0-data (this should never happen, but just in case) if bytesToFetch == 0 { innerIndex = 0 outerIndex++ continue } + if curIndex == 0 { + firstBytesToFetch = bytesToFetch + } + // If we've already placed some data in this chunk, that means + // a new data segment begins + if curIndex != 0 { + // Offset by the fixed reserved bytes at the beginning of the share + startIndex = firstBytesToFetch + NamespaceSize + ShareReservedBytes + } rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...) innerIndex += bytesToFetch if innerIndex >= len(rawDatas[outerIndex]) { @@ -140,8 +143,7 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex += bytesToFetch } - // Check if there is any more data, if not then we've reached the end - // and start index of next is 0 + // Prune any extra 0-data (this should never happen, but just in case) for outerIndex < len(rawDatas) { bytesToFetch := len(rawDatas[outerIndex]) - innerIndex if bytesToFetch == 0 { @@ -151,9 +153,6 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) } break } - if outerIndex >= len(rawDatas) { - startIndex = 0 - } return rawData, outerIndex, innerIndex, startIndex } From 92301ea32b7257db542443dfa9758fdebaacf3c6 Mon Sep 17 00:00:00 2001 From: John Adler Date: Wed, 24 Mar 2021 15:42:46 -0400 Subject: [PATCH 26/30] Update comments. --- types/shares.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/types/shares.go b/types/shares.go index 9a677c2568..8d8085d666 100644 --- a/types/shares.go +++ b/types/shares.go @@ -56,8 +56,8 @@ func (m Message) MarshalDelimited() ([]byte, error) { return append(lenBuf[:n], m.Data...), nil } -// appendToSharesContiguous appends one raw data separately as shares -// Used for messages +// appendToShares appends raw data as shares. +// Used for messages. func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { const adjustedSize = ShareSize - NamespaceSize if len(rawData) < adjustedSize { @@ -71,8 +71,8 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) return shares } -// splitContiguous splits multiple raw data contiguously as shares -// Used for transactions, intermediate state roots, and evidence +// splitContiguous splits multiple raw data contiguously as shares. +// Used for transactions, intermediate state roots, and evidence. func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { shares := make([]NamespacedShare, 0) const adjustedSize = ShareSize - NamespaceSize - ShareReservedBytes From b70b103cec9a967b42c7d9bdeeb5051628184418 Mon Sep 17 00:00:00 2001 From: John Adler Date: Thu, 25 Mar 2021 10:50:37 -0400 Subject: [PATCH 27/30] Make tx and msg adjusted share sizes exported constants. --- types/consts.go | 5 +++++ types/shares.go | 15 ++++++--------- types/shares_test.go | 22 ++++++++++------------ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/types/consts.go b/types/consts.go index e433f9048e..0fb328a9d7 100644 --- a/types/consts.go +++ b/types/consts.go @@ -19,6 +19,11 @@ const ( // ShareReservedBytes is the reserved bytes for contiguous appends. ShareReservedBytes = 1 + // TxShareSize is the number of bytes usable for tx/evidence/ISR shares. + TxShareSize = ShareSize - NamespaceSize - ShareReservedBytes + // MsgShareSize is the number of bytes usable for message shares. + MsgShareSize = ShareSize - NamespaceSize + // MaxSquareSize is the maximum number of // rows/columns of the original data shares in square layout. // Corresponds to AVAILABLE_DATA_ORIGINAL_SQUARE_MAX in the spec. diff --git a/types/shares.go b/types/shares.go index 8d8085d666..403ec0d734 100644 --- a/types/shares.go +++ b/types/shares.go @@ -59,13 +59,12 @@ func (m Message) MarshalDelimited() ([]byte, error) { // appendToShares appends raw data as shares. // Used for messages. func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { - const adjustedSize = ShareSize - NamespaceSize - if len(rawData) < adjustedSize { + if len(rawData) < MsgShareSize { rawShare := []byte(append(nid, rawData...)) paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} shares = append(shares, share) - } else { // len(rawData) >= adjustedSize + } else { // len(rawData) >= MsgShareSize shares = append(shares, split(rawData, nid)...) } return shares @@ -75,7 +74,6 @@ func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) // Used for transactions, intermediate state roots, and evidence. func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { shares := make([]NamespacedShare, 0) - const adjustedSize = ShareSize - NamespaceSize - ShareReservedBytes // Index into the outer slice of rawDatas outerIndex := 0 // Index into the inner slice of rawDatas @@ -83,7 +81,7 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { for outerIndex < len(rawDatas) { var rawData []byte startIndex := 0 - rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, adjustedSize) + rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize) rawShare := []byte(append(append(nid, byte(startIndex)), rawData...)) paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} @@ -95,13 +93,12 @@ func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare { // TODO(ismail): implement corresponding merge method for clients requesting // shares for a particular namespace func split(rawData []byte, nid namespace.ID) []NamespacedShare { - const adjustedSize = ShareSize - NamespaceSize shares := make([]NamespacedShare, 0) - firstRawShare := []byte(append(nid, rawData[:adjustedSize]...)) + firstRawShare := []byte(append(nid, rawData[:MsgShareSize]...)) shares = append(shares, NamespacedShare{firstRawShare, nid}) - rawData = rawData[adjustedSize:] + rawData = rawData[MsgShareSize:] for len(rawData) > 0 { - shareSizeOrLen := min(adjustedSize, len(rawData)) + shareSizeOrLen := min(MsgShareSize, len(rawData)) rawShare := []byte(append(nid, rawData[:shareSizeOrLen]...)) paddedShare := zeroPadIfNecessary(rawShare, ShareSize) share := NamespacedShare{paddedShare, nid} diff --git a/types/shares_test.go b/types/shares_test.go index b755a15ae7..b3d0c9c063 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -38,8 +38,6 @@ func TestMakeShares(t *testing.T) { if err != nil { t.Fatalf("Could not encode evidence: %v, error: %v\n", testEvidence, err) } - const txShareSize = ShareSize - NamespaceSize - ShareReservedBytes - const msgShareSize = ShareSize - NamespaceSize type args struct { data splitter @@ -57,13 +55,13 @@ func TestMakeShares(t *testing.T) { }, NamespacedShares{NamespacedShare{ Share: append( append(reservedEvidenceNamespaceID, byte(0)), - testEvidenceBytes[:txShareSize]..., + testEvidenceBytes[:TxShareSize]..., ), ID: reservedEvidenceNamespaceID, }, NamespacedShare{ Share: append( append(reservedEvidenceNamespaceID, byte(0)), - zeroPadIfNecessary(testEvidenceBytes[txShareSize:], txShareSize)..., + zeroPadIfNecessary(testEvidenceBytes[TxShareSize:], TxShareSize)..., ), ID: reservedEvidenceNamespaceID, }}, @@ -76,7 +74,7 @@ func TestMakeShares(t *testing.T) { NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), - zeroPadIfNecessary(smolTxLenDelimited, txShareSize)..., + zeroPadIfNecessary(smolTxLenDelimited, TxShareSize)..., ), ID: reservedTxNamespaceID, }, @@ -90,14 +88,14 @@ func TestMakeShares(t *testing.T) { NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), - largeTxLenDelimited[:txShareSize]..., + largeTxLenDelimited[:TxShareSize]..., ), ID: reservedTxNamespaceID, }, NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), - zeroPadIfNecessary(largeTxLenDelimited[txShareSize:], txShareSize)..., + zeroPadIfNecessary(largeTxLenDelimited[TxShareSize:], TxShareSize)..., ), ID: reservedTxNamespaceID, }, @@ -111,16 +109,16 @@ func TestMakeShares(t *testing.T) { NamespacedShare{ Share: append( append(reservedTxNamespaceID, byte(0)), - largeTxLenDelimited[:txShareSize]..., + largeTxLenDelimited[:TxShareSize]..., ), ID: reservedTxNamespaceID, }, NamespacedShare{ Share: append( - append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-txShareSize+NamespaceSize+ShareReservedBytes)), + append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-TxShareSize+NamespaceSize+ShareReservedBytes)), zeroPadIfNecessary( - append(largeTxLenDelimited[txShareSize:], smolTxLenDelimited...), - txShareSize, + append(largeTxLenDelimited[TxShareSize:], smolTxLenDelimited...), + TxShareSize, )..., ), ID: reservedTxNamespaceID, @@ -135,7 +133,7 @@ func TestMakeShares(t *testing.T) { NamespacedShare{ Share: append( []byte(msg1.NamespaceID), - zeroPadIfNecessary(msg1Marshaled, msgShareSize)..., + zeroPadIfNecessary(msg1Marshaled, MsgShareSize)..., ), ID: msg1.NamespaceID, }, From ff636c4445db7ece6dcc764d4bd3628e6c4c1161 Mon Sep 17 00:00:00 2001 From: John Adler Date: Thu, 25 Mar 2021 10:54:48 -0400 Subject: [PATCH 28/30] Panic on hopefully-unreachable condition instead of silently skipping. --- types/shares.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/types/shares.go b/types/shares.go index 403ec0d734..5a7a4a0896 100644 --- a/types/shares.go +++ b/types/shares.go @@ -3,6 +3,7 @@ package types import ( "bytes" "encoding/binary" + "fmt" "github.com/lazyledger/nmt/namespace" ) @@ -108,6 +109,9 @@ func split(rawData []byte, nid namespace.ID) []NamespacedShare { return shares } +// getNextChunk gets the next chunk for contiguous shares +// Precondition: none of the slices in rawDatas is zero-length +// This precondition should always hold at this point since zero-length txs are simply invalid. func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) { rawData := make([]byte, 0, width) startIndex := 0 @@ -116,11 +120,8 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex := 0 for curIndex < width && outerIndex < len(rawDatas) { bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex) - // Prune any extra 0-data (this should never happen, but just in case) if bytesToFetch == 0 { - innerIndex = 0 - outerIndex++ - continue + panic(fmt.Sprintf("zero-length contiguous share data is invalid")) } if curIndex == 0 { firstBytesToFetch = bytesToFetch @@ -140,17 +141,6 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) curIndex += bytesToFetch } - // Prune any extra 0-data (this should never happen, but just in case) - for outerIndex < len(rawDatas) { - bytesToFetch := len(rawDatas[outerIndex]) - innerIndex - if bytesToFetch == 0 { - innerIndex = 0 - outerIndex++ - continue - } - break - } - return rawData, outerIndex, innerIndex, startIndex } From 0a731c9b95c90e5d25843d815c7109d7cdb08171 Mon Sep 17 00:00:00 2001 From: John Adler Date: Thu, 25 Mar 2021 10:59:06 -0400 Subject: [PATCH 29/30] Fix lint. --- types/shares.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/types/shares.go b/types/shares.go index 5a7a4a0896..6b64cb6cec 100644 --- a/types/shares.go +++ b/types/shares.go @@ -3,7 +3,6 @@ package types import ( "bytes" "encoding/binary" - "fmt" "github.com/lazyledger/nmt/namespace" ) @@ -121,7 +120,7 @@ func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) for curIndex < width && outerIndex < len(rawDatas) { bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex) if bytesToFetch == 0 { - panic(fmt.Sprintf("zero-length contiguous share data is invalid")) + panic("zero-length contiguous share data is invalid") } if curIndex == 0 { firstBytesToFetch = bytesToFetch From 3eb7223d287d3b1794e966c565b38acd119cb9b5 Mon Sep 17 00:00:00 2001 From: John Adler Date: Thu, 25 Mar 2021 11:08:33 -0400 Subject: [PATCH 30/30] Update hardcoded response for block format. --- blockchain/msgs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go index 818c4cb2cb..7c308f9657 100644 --- a/blockchain/msgs_test.go +++ b/blockchain/msgs_test.go @@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) { BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}}, "0a0a08ffffffffffffffff7f"}, {"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{ - BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a2016106406aededa633a265efd5833af53775af5a27c803518313c9b77e68875926a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c10a30ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07123000000000000000010000000000000001266fcca850585e31b54074399b0dbb5b04b408b77d5f357a528ed3e04b63c3c11230ffffffffffffffffffffffffffffffffe04e02811b28234428f1c1219d5627f61c02c415c175bdabd9e7934519baef07"}, + BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac0020abd020a5b0a02080b1803220b088092b8c398feffffff012a0212003a204c149a7cfadc92b669b0cbfa4951a1b18c2d9f3177a3b8756d39ebb96e9d63316a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a0a30ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7123000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a1230ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7"}, {"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{ NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"}, {"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{