diff --git a/pkg/cnservice/server.go b/pkg/cnservice/server.go index 521a1a838165..69360ef87e7d 100644 --- a/pkg/cnservice/server.go +++ b/pkg/cnservice/server.go @@ -25,6 +25,8 @@ import ( "github.com/fagongzi/goetty/v2" "github.com/google/uuid" + "go.uber.org/zap" + "github.com/matrixorigin/matrixone/pkg/bootstrap" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -66,7 +68,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio" "github.com/matrixorigin/matrixone/pkg/vm/process" - "go.uber.org/zap" ) func NewService( @@ -764,10 +765,13 @@ func (s *service) initShardService() { shardservice.ReadApproxObjectsNum: disttae.HandleShardingReadApproxObjectsNum, shardservice.ReadRanges: disttae.HandleShardingReadRanges, shardservice.ReadGetColumMetadataScanInfo: disttae.HandleShardingReadGetColumMetadataScanInfo, - shardservice.ReadReader: disttae.HandleShardingReadReader, + shardservice.ReadBuildReader: disttae.HandleShardingReadBuildReader, shardservice.ReadPrimaryKeysMayBeModified: disttae.HandleShardingReadPrimaryKeysMayBeModified, shardservice.ReadMergeObjects: disttae.HandleShardingReadMergeObjects, shardservice.ReadVisibleObjectStats: disttae.HandleShardingReadVisibleObjectStats, + shardservice.ReadClose: disttae.HandleShardingReadClose, + shardservice.ReadNext: disttae.HandleShardingReadNext, + shardservice.ReadCollectTombstones: disttae.HandleShardingReadCollectTombstones, }, s.storeEngine, ) diff --git a/pkg/common/morpc/buffer_pool.go b/pkg/common/morpc/buffer_pool.go index 90f3a59c35ed..b1a022ad2937 100644 --- a/pkg/common/morpc/buffer_pool.go +++ b/pkg/common/morpc/buffer_pool.go @@ -16,6 +16,7 @@ package morpc import ( "github.com/fagongzi/goetty/v2/buf" + "github.com/matrixorigin/matrixone/pkg/common/reuse" ) @@ -91,5 +92,8 @@ func (b *Buffer) Mark() { } func (b *Buffer) GetMarkedData() []byte { + if b.markIdx == b.buf.GetWriteIndex() { + return nil + } return b.buf.RawSlice(b.markIdx, b.buf.GetWriteIndex()) } diff --git a/pkg/frontend/test/engine_mock.go b/pkg/frontend/test/engine_mock.go index 4861a4801a2f..8cc31bc7c8dd 100644 --- a/pkg/frontend/test/engine_mock.go +++ b/pkg/frontend/test/engine_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: ../../../pkg/vm/engine/types.go +// Source: pkg/vm/engine/types.go // Package mock_frontend is a generated GoMock package. package mock_frontend @@ -229,18 +229,18 @@ func (mr *MockTombstonerMockRecorder) ApplyInMemTombstones(bid, rowsOffset, dele } // ApplyPersistedTombstones mocks base method. -func (m *MockTombstoner) ApplyPersistedTombstones(ctx context.Context, bid types.Blockid, rowsOffset []int64, mask *nulls.Nulls, apply func(context.Context, objectio.Location, types.TS, []int64, *nulls.Nulls) ([]int64, error)) ([]int64, error) { +func (m *MockTombstoner) ApplyPersistedTombstones(ctx context.Context, fs fileservice.FileService, snapshot types.TS, bid types.Blockid, rowsOffset []int64, deletedMask *nulls.Nulls) ([]int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyPersistedTombstones", ctx, bid, rowsOffset, mask, apply) + ret := m.ctrl.Call(m, "ApplyPersistedTombstones", ctx, fs, snapshot, bid, rowsOffset, deletedMask) ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ApplyPersistedTombstones indicates an expected call of ApplyPersistedTombstones. -func (mr *MockTombstonerMockRecorder) ApplyPersistedTombstones(ctx, bid, rowsOffset, mask, apply interface{}) *gomock.Call { +func (mr *MockTombstonerMockRecorder) ApplyPersistedTombstones(ctx, fs, snapshot, bid, rowsOffset, deletedMask interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyPersistedTombstones", reflect.TypeOf((*MockTombstoner)(nil).ApplyPersistedTombstones), ctx, bid, rowsOffset, mask, apply) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyPersistedTombstones", reflect.TypeOf((*MockTombstoner)(nil).ApplyPersistedTombstones), ctx, fs, snapshot, bid, rowsOffset, deletedMask) } // HasAnyInMemoryTombstone mocks base method. @@ -686,18 +686,18 @@ func (m *MockDataSource) EXPECT() *MockDataSourceMockRecorder { } // ApplyTombstones mocks base method. -func (m *MockDataSource) ApplyTombstones(ctx context.Context, bid objectio.Blockid, rowsOffset []int64) ([]int64, error) { +func (m *MockDataSource) ApplyTombstones(ctx context.Context, bid objectio.Blockid, rowsOffset []int64, applyPolicy engine.TombstoneApplyPolicy) ([]int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyTombstones", ctx, bid, rowsOffset) + ret := m.ctrl.Call(m, "ApplyTombstones", ctx, bid, rowsOffset, applyPolicy) ret0, _ := ret[0].([]int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ApplyTombstones indicates an expected call of ApplyTombstones. -func (mr *MockDataSourceMockRecorder) ApplyTombstones(ctx, bid, rowsOffset interface{}) *gomock.Call { +func (mr *MockDataSourceMockRecorder) ApplyTombstones(ctx, bid, rowsOffset, applyPolicy interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyTombstones", reflect.TypeOf((*MockDataSource)(nil).ApplyTombstones), ctx, bid, rowsOffset) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyTombstones", reflect.TypeOf((*MockDataSource)(nil).ApplyTombstones), ctx, bid, rowsOffset, applyPolicy) } // Close mocks base method. @@ -742,9 +742,9 @@ func (mr *MockDataSourceMockRecorder) GetTombstones(ctx, bid interface{}) *gomoc } // Next mocks base method. -func (m *MockDataSource) Next(ctx context.Context, cols []string, types []types.Type, seqNums []uint16, memFilter any, mp *mpool.MPool, vp engine.VectorPool, bat *batch.Batch) (*objectio.BlockInfo, engine.DataState, error) { +func (m *MockDataSource) Next(ctx context.Context, cols []string, types []types.Type, seqNums []uint16, memFilter any, mp *mpool.MPool, bat *batch.Batch) (*objectio.BlockInfo, engine.DataState, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Next", ctx, cols, types, seqNums, memFilter, mp, vp, bat) + ret := m.ctrl.Call(m, "Next", ctx, cols, types, seqNums, memFilter, mp, bat) ret0, _ := ret[0].(*objectio.BlockInfo) ret1, _ := ret[1].(engine.DataState) ret2, _ := ret[2].(error) @@ -752,9 +752,9 @@ func (m *MockDataSource) Next(ctx context.Context, cols []string, types []types. } // Next indicates an expected call of Next. -func (mr *MockDataSourceMockRecorder) Next(ctx, cols, types, seqNums, memFilter, mp, vp, bat interface{}) *gomock.Call { +func (mr *MockDataSourceMockRecorder) Next(ctx, cols, types, seqNums, memFilter, mp, bat interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockDataSource)(nil).Next), ctx, cols, types, seqNums, memFilter, mp, vp, bat) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockDataSource)(nil).Next), ctx, cols, types, seqNums, memFilter, mp, bat) } // SetFilterZM mocks base method. @@ -898,6 +898,60 @@ func (mr *MockRangesMockRecorder) Slice(i, j interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Slice", reflect.TypeOf((*MockRanges)(nil).Slice), i, j) } +// MockChangesHandle is a mock of ChangesHandle interface. +type MockChangesHandle struct { + ctrl *gomock.Controller + recorder *MockChangesHandleMockRecorder +} + +// MockChangesHandleMockRecorder is the mock recorder for MockChangesHandle. +type MockChangesHandleMockRecorder struct { + mock *MockChangesHandle +} + +// NewMockChangesHandle creates a new mock instance. +func NewMockChangesHandle(ctrl *gomock.Controller) *MockChangesHandle { + mock := &MockChangesHandle{ctrl: ctrl} + mock.recorder = &MockChangesHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockChangesHandle) EXPECT() *MockChangesHandleMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockChangesHandle) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockChangesHandleMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockChangesHandle)(nil).Close)) +} + +// Next mocks base method. +func (m *MockChangesHandle) Next(ctx context.Context, mp *mpool.MPool) (*batch.Batch, *batch.Batch, engine.ChangesHandle_Hint, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Next", ctx, mp) + ret0, _ := ret[0].(*batch.Batch) + ret1, _ := ret[1].(*batch.Batch) + ret2, _ := ret[2].(engine.ChangesHandle_Hint) + ret3, _ := ret[3].(error) + return ret0, ret1, ret2, ret3 +} + +// Next indicates an expected call of Next. +func (mr *MockChangesHandleMockRecorder) Next(ctx, mp interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockChangesHandle)(nil).Next), ctx, mp) +} + // MockRelation is a mock of Relation interface. type MockRelation struct { ctrl *gomock.Controller @@ -916,10 +970,6 @@ func NewMockRelation(ctrl *gomock.Controller) *MockRelation { return mock } -func (m *MockRelation) CollectChanges(_ context.Context, from, to types.TS, _ *mpool.MPool) (engine.ChangesHandle, error) { - panic("not support") -} - // EXPECT returns an object that allows the caller to indicate expected use. func (m *MockRelation) EXPECT() *MockRelationMockRecorder { return m.recorder @@ -982,19 +1032,49 @@ func (mr *MockRelationMockRecorder) BuildReaders(ctx, proc, expr, relData, num, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildReaders", reflect.TypeOf((*MockRelation)(nil).BuildReaders), ctx, proc, expr, relData, num, txnOffset, orderBy, policy) } +// BuildShardingReaders mocks base method. +func (m *MockRelation) BuildShardingReaders(ctx context.Context, proc any, expr *plan.Expr, relData engine.RelData, num, txnOffset int, orderBy bool, policy engine.TombstoneApplyPolicy) ([]engine.Reader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BuildShardingReaders", ctx, proc, expr, relData, num, txnOffset, orderBy, policy) + ret0, _ := ret[0].([]engine.Reader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BuildShardingReaders indicates an expected call of BuildShardingReaders. +func (mr *MockRelationMockRecorder) BuildShardingReaders(ctx, proc, expr, relData, num, txnOffset, orderBy, policy interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildShardingReaders", reflect.TypeOf((*MockRelation)(nil).BuildShardingReaders), ctx, proc, expr, relData, num, txnOffset, orderBy, policy) +} + +// CollectChanges mocks base method. +func (m *MockRelation) CollectChanges(ctx context.Context, from, to types.TS, mp *mpool.MPool) (engine.ChangesHandle, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CollectChanges", ctx, from, to, mp) + ret0, _ := ret[0].(engine.ChangesHandle) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CollectChanges indicates an expected call of CollectChanges. +func (mr *MockRelationMockRecorder) CollectChanges(ctx, from, to, mp interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CollectChanges", reflect.TypeOf((*MockRelation)(nil).CollectChanges), ctx, from, to, mp) +} + // CollectTombstones mocks base method. -func (m *MockRelation) CollectTombstones(ctx context.Context, txnOffset int) (engine.Tombstoner, error) { +func (m *MockRelation) CollectTombstones(ctx context.Context, txnOffset int, policy engine.TombstoneCollectPolicy) (engine.Tombstoner, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CollectTombstones", ctx, txnOffset) + ret := m.ctrl.Call(m, "CollectTombstones", ctx, txnOffset, policy) ret0, _ := ret[0].(engine.Tombstoner) ret1, _ := ret[1].(error) return ret0, ret1 } // CollectTombstones indicates an expected call of CollectTombstones. -func (mr *MockRelationMockRecorder) CollectTombstones(ctx, txnOffset interface{}) *gomock.Call { +func (mr *MockRelationMockRecorder) CollectTombstones(ctx, txnOffset, policy interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CollectTombstones", reflect.TypeOf((*MockRelation)(nil).CollectTombstones), ctx, txnOffset) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CollectTombstones", reflect.TypeOf((*MockRelation)(nil).CollectTombstones), ctx, txnOffset, policy) } // CopyTableDef mocks base method. @@ -1127,6 +1207,20 @@ func (mr *MockRelationMockRecorder) GetPrimaryKeys(arg0 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrimaryKeys", reflect.TypeOf((*MockRelation)(nil).GetPrimaryKeys), arg0) } +// GetProcess mocks base method. +func (m *MockRelation) GetProcess() any { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProcess") + ret0, _ := ret[0].(any) + return ret0 +} + +// GetProcess indicates an expected call of GetProcess. +func (mr *MockRelationMockRecorder) GetProcess() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProcess", reflect.TypeOf((*MockRelation)(nil).GetProcess)) +} + // GetTableDef mocks base method. func (m *MockRelation) GetTableDef(arg0 context.Context) *plan.TableDef { m.ctrl.T.Helper() diff --git a/pkg/pb/shard/shard.pb.go b/pkg/pb/shard/shard.pb.go index b74a2e55fcdb..f3515e2d1528 100644 --- a/pkg/pb/shard/shard.pb.go +++ b/pkg/pb/shard/shard.pb.go @@ -1466,8 +1466,11 @@ type ReadParam struct { GetColumMetadataScanInfoParam GetColumMetadataScanInfoParam `protobuf:"bytes,6,opt,name=GetColumMetadataScanInfoParam,proto3" json:"GetColumMetadataScanInfoParam"` MergeObjectsParam MergeObjectsParam `protobuf:"bytes,7,opt,name=MergeObjectsParam,proto3" json:"MergeObjectsParam"` KeyParam KeyParam `protobuf:"bytes,8,opt,name=KeyParam,proto3" json:"KeyParam"` - ReaderParam ReaderParam `protobuf:"bytes,9,opt,name=ReaderParam,proto3" json:"ReaderParam"` + ReaderBuildParam ReaderBuildParam `protobuf:"bytes,9,opt,name=ReaderBuildParam,proto3" json:"ReaderBuildParam"` PrimaryKeysMayBeModifiedParam PrimaryKeysMayBeModifiedParam `protobuf:"bytes,10,opt,name=PrimaryKeysMayBeModifiedParam,proto3" json:"PrimaryKeysMayBeModifiedParam"` + ReadNextParam ReadNextParam `protobuf:"bytes,11,opt,name=ReadNextParam,proto3" json:"ReadNextParam"` + ReadCloseParam ReadCloseParam `protobuf:"bytes,12,opt,name=ReadCloseParam,proto3" json:"ReadCloseParam"` + CollectTombstonesParam CollectTombstonesParam `protobuf:"bytes,13,opt,name=CollectTombstonesParam,proto3" json:"CollectTombstonesParam"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1562,11 +1565,11 @@ func (m *ReadParam) GetKeyParam() KeyParam { return KeyParam{} } -func (m *ReadParam) GetReaderParam() ReaderParam { +func (m *ReadParam) GetReaderBuildParam() ReaderBuildParam { if m != nil { - return m.ReaderParam + return m.ReaderBuildParam } - return ReaderParam{} + return ReaderBuildParam{} } func (m *ReadParam) GetPrimaryKeysMayBeModifiedParam() PrimaryKeysMayBeModifiedParam { @@ -1576,6 +1579,27 @@ func (m *ReadParam) GetPrimaryKeysMayBeModifiedParam() PrimaryKeysMayBeModifiedP return PrimaryKeysMayBeModifiedParam{} } +func (m *ReadParam) GetReadNextParam() ReadNextParam { + if m != nil { + return m.ReadNextParam + } + return ReadNextParam{} +} + +func (m *ReadParam) GetReadCloseParam() ReadCloseParam { + if m != nil { + return m.ReadCloseParam + } + return ReadCloseParam{} +} + +func (m *ReadParam) GetCollectTombstonesParam() CollectTombstonesParam { + if m != nil { + return m.CollectTombstonesParam + } + return CollectTombstonesParam{} +} + type TxnTable struct { AccountID uint64 `protobuf:"varint,1,opt,name=AccountID,proto3" json:"AccountID,omitempty"` DatabaseID uint64 `protobuf:"varint,2,opt,name=DatabaseID,proto3" json:"DatabaseID,omitempty"` @@ -1796,6 +1820,53 @@ func (m *RangesParam) GetUncommittedObjects() []byte { return nil } +type CollectTombstonesParam struct { + CollectPolicy uint64 `protobuf:"varint,1,opt,name=collectPolicy,proto3" json:"collectPolicy,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CollectTombstonesParam) Reset() { *m = CollectTombstonesParam{} } +func (m *CollectTombstonesParam) String() string { return proto.CompactTextString(m) } +func (*CollectTombstonesParam) ProtoMessage() {} +func (*CollectTombstonesParam) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{25} +} +func (m *CollectTombstonesParam) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CollectTombstonesParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CollectTombstonesParam.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CollectTombstonesParam) XXX_Merge(src proto.Message) { + xxx_messageInfo_CollectTombstonesParam.Merge(m, src) +} +func (m *CollectTombstonesParam) XXX_Size() int { + return m.ProtoSize() +} +func (m *CollectTombstonesParam) XXX_DiscardUnknown() { + xxx_messageInfo_CollectTombstonesParam.DiscardUnknown(m) +} + +var xxx_messageInfo_CollectTombstonesParam proto.InternalMessageInfo + +func (m *CollectTombstonesParam) GetCollectPolicy() uint64 { + if m != nil { + return m.CollectPolicy + } + return 0 +} + type GetColumMetadataScanInfoParam struct { ColumnName string `protobuf:"bytes,1,opt,name=columnName,proto3" json:"columnName,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1807,7 +1878,7 @@ func (m *GetColumMetadataScanInfoParam) Reset() { *m = GetColumMetadataS func (m *GetColumMetadataScanInfoParam) String() string { return proto.CompactTextString(m) } func (*GetColumMetadataScanInfoParam) ProtoMessage() {} func (*GetColumMetadataScanInfoParam) Descriptor() ([]byte, []int) { - return fileDescriptor_319ea41e44cdc364, []int{25} + return fileDescriptor_319ea41e44cdc364, []int{26} } func (m *GetColumMetadataScanInfoParam) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1854,7 +1925,7 @@ func (m *KeyParam) Reset() { *m = KeyParam{} } func (m *KeyParam) String() string { return proto.CompactTextString(m) } func (*KeyParam) ProtoMessage() {} func (*KeyParam) Descriptor() ([]byte, []int) { - return fileDescriptor_319ea41e44cdc364, []int{26} + return fileDescriptor_319ea41e44cdc364, []int{27} } func (m *KeyParam) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1890,29 +1961,28 @@ func (m *KeyParam) GetKey() []byte { return nil } -type ReaderParam struct { - Num int32 `protobuf:"varint,1,opt,name=num,proto3" json:"num,omitempty"` - Expr plan.Expr `protobuf:"bytes,2,opt,name=expr,proto3" json:"expr"` - Ranges []byte `protobuf:"bytes,3,opt,name=ranges,proto3" json:"ranges,omitempty"` - OrderedScan bool `protobuf:"varint,4,opt,name=orderedScan,proto3" json:"orderedScan,omitempty"` - TxnOffset int32 `protobuf:"varint,5,opt,name=txnOffset,proto3" json:"txnOffset,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type ReaderBuildParam struct { + RelData []byte `protobuf:"bytes,1,opt,name=relData,proto3" json:"relData,omitempty"` + Expr *plan.Expr `protobuf:"bytes,2,opt,name=expr,proto3" json:"expr,omitempty"` + ScanType int32 `protobuf:"varint,3,opt,name=scanType,proto3" json:"scanType,omitempty"` + TombstoneApplyPolicy int32 `protobuf:"varint,4,opt,name=tombstoneApplyPolicy,proto3" json:"tombstoneApplyPolicy,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ReaderParam) Reset() { *m = ReaderParam{} } -func (m *ReaderParam) String() string { return proto.CompactTextString(m) } -func (*ReaderParam) ProtoMessage() {} -func (*ReaderParam) Descriptor() ([]byte, []int) { - return fileDescriptor_319ea41e44cdc364, []int{27} +func (m *ReaderBuildParam) Reset() { *m = ReaderBuildParam{} } +func (m *ReaderBuildParam) String() string { return proto.CompactTextString(m) } +func (*ReaderBuildParam) ProtoMessage() {} +func (*ReaderBuildParam) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{28} } -func (m *ReaderParam) XXX_Unmarshal(b []byte) error { +func (m *ReaderBuildParam) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *ReaderParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *ReaderBuildParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_ReaderParam.Marshal(b, m, deterministic) + return xxx_messageInfo_ReaderBuildParam.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -1922,51 +1992,240 @@ func (m *ReaderParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return b[:n], nil } } -func (m *ReaderParam) XXX_Merge(src proto.Message) { - xxx_messageInfo_ReaderParam.Merge(m, src) +func (m *ReaderBuildParam) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReaderBuildParam.Merge(m, src) } -func (m *ReaderParam) XXX_Size() int { +func (m *ReaderBuildParam) XXX_Size() int { return m.ProtoSize() } -func (m *ReaderParam) XXX_DiscardUnknown() { - xxx_messageInfo_ReaderParam.DiscardUnknown(m) +func (m *ReaderBuildParam) XXX_DiscardUnknown() { + xxx_messageInfo_ReaderBuildParam.DiscardUnknown(m) } -var xxx_messageInfo_ReaderParam proto.InternalMessageInfo +var xxx_messageInfo_ReaderBuildParam proto.InternalMessageInfo -func (m *ReaderParam) GetNum() int32 { +func (m *ReaderBuildParam) GetRelData() []byte { if m != nil { - return m.Num + return m.RelData } - return 0 + return nil } -func (m *ReaderParam) GetExpr() plan.Expr { +func (m *ReaderBuildParam) GetExpr() *plan.Expr { if m != nil { return m.Expr } - return plan.Expr{} + return nil +} + +func (m *ReaderBuildParam) GetScanType() int32 { + if m != nil { + return m.ScanType + } + return 0 +} + +func (m *ReaderBuildParam) GetTombstoneApplyPolicy() int32 { + if m != nil { + return m.TombstoneApplyPolicy + } + return 0 +} + +type ReaderBuildResult struct { + Uuid []byte `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReaderBuildResult) Reset() { *m = ReaderBuildResult{} } +func (m *ReaderBuildResult) String() string { return proto.CompactTextString(m) } +func (*ReaderBuildResult) ProtoMessage() {} +func (*ReaderBuildResult) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{29} +} +func (m *ReaderBuildResult) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ReaderBuildResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ReaderBuildResult.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ReaderBuildResult) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReaderBuildResult.Merge(m, src) +} +func (m *ReaderBuildResult) XXX_Size() int { + return m.ProtoSize() +} +func (m *ReaderBuildResult) XXX_DiscardUnknown() { + xxx_messageInfo_ReaderBuildResult.DiscardUnknown(m) } -func (m *ReaderParam) GetRanges() []byte { +var xxx_messageInfo_ReaderBuildResult proto.InternalMessageInfo + +func (m *ReaderBuildResult) GetUuid() []byte { if m != nil { - return m.Ranges + return m.Uuid } return nil } -func (m *ReaderParam) GetOrderedScan() bool { +type ReadNextParam struct { + Uuid []byte `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` + Columns []string `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReadNextParam) Reset() { *m = ReadNextParam{} } +func (m *ReadNextParam) String() string { return proto.CompactTextString(m) } +func (*ReadNextParam) ProtoMessage() {} +func (*ReadNextParam) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{30} +} +func (m *ReadNextParam) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ReadNextParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ReadNextParam.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ReadNextParam) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReadNextParam.Merge(m, src) +} +func (m *ReadNextParam) XXX_Size() int { + return m.ProtoSize() +} +func (m *ReadNextParam) XXX_DiscardUnknown() { + xxx_messageInfo_ReadNextParam.DiscardUnknown(m) +} + +var xxx_messageInfo_ReadNextParam proto.InternalMessageInfo + +func (m *ReadNextParam) GetUuid() []byte { if m != nil { - return m.OrderedScan + return m.Uuid } - return false + return nil } -func (m *ReaderParam) GetTxnOffset() int32 { +func (m *ReadNextParam) GetColumns() []string { if m != nil { - return m.TxnOffset + return m.Columns } - return 0 + return nil +} + +type ReadNextResult struct { + Result []byte `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReadNextResult) Reset() { *m = ReadNextResult{} } +func (m *ReadNextResult) String() string { return proto.CompactTextString(m) } +func (*ReadNextResult) ProtoMessage() {} +func (*ReadNextResult) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{31} +} +func (m *ReadNextResult) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ReadNextResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ReadNextResult.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ReadNextResult) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReadNextResult.Merge(m, src) +} +func (m *ReadNextResult) XXX_Size() int { + return m.ProtoSize() +} +func (m *ReadNextResult) XXX_DiscardUnknown() { + xxx_messageInfo_ReadNextResult.DiscardUnknown(m) +} + +var xxx_messageInfo_ReadNextResult proto.InternalMessageInfo + +func (m *ReadNextResult) GetResult() []byte { + if m != nil { + return m.Result + } + return nil +} + +type ReadCloseParam struct { + Uuid []byte `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReadCloseParam) Reset() { *m = ReadCloseParam{} } +func (m *ReadCloseParam) String() string { return proto.CompactTextString(m) } +func (*ReadCloseParam) ProtoMessage() {} +func (*ReadCloseParam) Descriptor() ([]byte, []int) { + return fileDescriptor_319ea41e44cdc364, []int{32} +} +func (m *ReadCloseParam) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ReadCloseParam) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ReadCloseParam.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ReadCloseParam) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReadCloseParam.Merge(m, src) +} +func (m *ReadCloseParam) XXX_Size() int { + return m.ProtoSize() +} +func (m *ReadCloseParam) XXX_DiscardUnknown() { + xxx_messageInfo_ReadCloseParam.DiscardUnknown(m) +} + +var xxx_messageInfo_ReadCloseParam proto.InternalMessageInfo + +func (m *ReadCloseParam) GetUuid() []byte { + if m != nil { + return m.Uuid + } + return nil } type PrimaryKeysMayBeModifiedParam struct { @@ -1982,7 +2241,7 @@ func (m *PrimaryKeysMayBeModifiedParam) Reset() { *m = PrimaryKeysMayBeM func (m *PrimaryKeysMayBeModifiedParam) String() string { return proto.CompactTextString(m) } func (*PrimaryKeysMayBeModifiedParam) ProtoMessage() {} func (*PrimaryKeysMayBeModifiedParam) Descriptor() ([]byte, []int) { - return fileDescriptor_319ea41e44cdc364, []int{28} + return fileDescriptor_319ea41e44cdc364, []int{33} } func (m *PrimaryKeysMayBeModifiedParam) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2044,7 +2303,7 @@ func (m *MergeObjectsParam) Reset() { *m = MergeObjectsParam{} } func (m *MergeObjectsParam) String() string { return proto.CompactTextString(m) } func (*MergeObjectsParam) ProtoMessage() {} func (*MergeObjectsParam) Descriptor() ([]byte, []int) { - return fileDescriptor_319ea41e44cdc364, []int{29} + return fileDescriptor_319ea41e44cdc364, []int{34} } func (m *MergeObjectsParam) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2118,9 +2377,14 @@ func init() { proto.RegisterType((*StatsParam)(nil), "shard.StatsParam") proto.RegisterType((*SizeParam)(nil), "shard.SizeParam") proto.RegisterType((*RangesParam)(nil), "shard.RangesParam") + proto.RegisterType((*CollectTombstonesParam)(nil), "shard.CollectTombstonesParam") proto.RegisterType((*GetColumMetadataScanInfoParam)(nil), "shard.GetColumMetadataScanInfoParam") proto.RegisterType((*KeyParam)(nil), "shard.KeyParam") - proto.RegisterType((*ReaderParam)(nil), "shard.ReaderParam") + proto.RegisterType((*ReaderBuildParam)(nil), "shard.ReaderBuildParam") + proto.RegisterType((*ReaderBuildResult)(nil), "shard.ReaderBuildResult") + proto.RegisterType((*ReadNextParam)(nil), "shard.ReadNextParam") + proto.RegisterType((*ReadNextResult)(nil), "shard.ReadNextResult") + proto.RegisterType((*ReadCloseParam)(nil), "shard.ReadCloseParam") proto.RegisterType((*PrimaryKeysMayBeModifiedParam)(nil), "shard.PrimaryKeysMayBeModifiedParam") proto.RegisterType((*MergeObjectsParam)(nil), "shard.MergeObjectsParam") } @@ -2128,112 +2392,121 @@ func init() { func init() { proto.RegisterFile("shard.proto", fileDescriptor_319ea41e44cdc364) } var fileDescriptor_319ea41e44cdc364 = []byte{ - // 1678 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x6f, 0x1b, 0xc5, - 0x17, 0xcf, 0xda, 0x6b, 0xc7, 0x7e, 0x76, 0x92, 0xcd, 0x34, 0xe9, 0xd7, 0xca, 0xb7, 0x04, 0xb3, - 0x6a, 0xab, 0x90, 0x52, 0x47, 0xa4, 0x94, 0x4a, 0x50, 0x09, 0xa5, 0x71, 0xd4, 0x44, 0x21, 0x3f, - 0xb4, 0x49, 0x7a, 0x00, 0x09, 0x34, 0xb6, 0x27, 0xce, 0xb6, 0xf6, 0xee, 0xb2, 0xbb, 0x86, 0x98, - 0x1b, 0x57, 0xfe, 0x01, 0x4e, 0x48, 0xfc, 0x13, 0x48, 0x5c, 0xb9, 0x55, 0xe2, 0xd2, 0x1b, 0x37, - 0x04, 0xe5, 0x1f, 0x41, 0xf3, 0x73, 0x67, 0xd6, 0x6e, 0xda, 0x43, 0x6f, 0xfb, 0x7e, 0xbf, 0x79, - 0xef, 0xe3, 0xf7, 0x66, 0x0c, 0xb5, 0xe4, 0x02, 0xc7, 0xbd, 0x56, 0x14, 0x87, 0x69, 0x88, 0x4a, - 0x8c, 0x58, 0xb9, 0xdb, 0xf7, 0xd3, 0x8b, 0x51, 0xa7, 0xd5, 0x0d, 0x87, 0x1b, 0xfd, 0xb0, 0x1f, - 0x6e, 0x30, 0x69, 0x67, 0x74, 0xce, 0x28, 0x46, 0xb0, 0x2f, 0x6e, 0xb5, 0xb2, 0x90, 0xfa, 0x43, - 0x92, 0xa4, 0x78, 0x18, 0x09, 0xc6, 0x7c, 0xe4, 0x47, 0x64, 0xe0, 0x07, 0x44, 0xd0, 0x10, 0x0d, - 0x70, 0xc0, 0xbf, 0xdd, 0x3f, 0x2d, 0x98, 0x3f, 0xa1, 0x51, 0x92, 0x03, 0x92, 0xe2, 0x1e, 0x4e, - 0x31, 0xba, 0x05, 0xe5, 0xe3, 0x70, 0xe0, 0x77, 0xc7, 0x0d, 0xab, 0x69, 0xad, 0xcd, 0x6f, 0xce, - 0xb5, 0x78, 0x4e, 0x9c, 0xe9, 0x09, 0x21, 0x6a, 0x42, 0x8d, 0x1b, 0x6e, 0x87, 0xa3, 0x20, 0x6d, - 0x14, 0x9a, 0xd6, 0xda, 0x9c, 0xa7, 0xb3, 0x50, 0x03, 0x66, 0x9f, 0x90, 0x38, 0xf1, 0xc3, 0xa0, - 0x51, 0x64, 0x52, 0x49, 0xa2, 0x1b, 0x50, 0xdd, 0xea, 0x76, 0xa9, 0xd2, 0x5e, 0xbb, 0x61, 0x37, - 0xad, 0x35, 0xdb, 0xcb, 0x18, 0x68, 0x0d, 0x16, 0x0e, 0xf0, 0xa5, 0x47, 0xa2, 0x81, 0xdf, 0xc5, - 0xdc, 0x7b, 0x89, 0xd9, 0xe7, 0xd9, 0x68, 0x05, 0x2a, 0x2c, 0xe0, 0x5e, 0x3b, 0x69, 0x94, 0x9b, - 0xc5, 0x35, 0xdb, 0x53, 0xb4, 0xfb, 0x9b, 0x05, 0x70, 0x8a, 0x3b, 0x03, 0xc2, 0x38, 0x34, 0x19, - 0x46, 0xed, 0xb5, 0xd9, 0xb1, 0x6c, 0x4f, 0x92, 0x54, 0x22, 0x8c, 0xd8, 0x21, 0x6c, 0x4f, 0x92, - 0x5a, 0x25, 0x8a, 0x57, 0x55, 0x42, 0x3b, 0xa7, 0x6d, 0x9e, 0xf3, 0x3e, 0x54, 0x44, 0xbe, 0x49, - 0xa3, 0xd4, 0x2c, 0xae, 0xd5, 0x36, 0xaf, 0x09, 0x17, 0x2c, 0x84, 0x90, 0x3d, 0xb2, 0x9f, 0xff, - 0xf5, 0xee, 0x8c, 0xa7, 0x54, 0xdd, 0x1f, 0x2c, 0xa8, 0xeb, 0x0a, 0xb4, 0x5e, 0xe2, 0x53, 0xa5, - 0x9f, 0x31, 0xd0, 0xfb, 0x50, 0x3a, 0x49, 0x71, 0x4a, 0x58, 0xfa, 0xf3, 0x2a, 0x84, 0x50, 0x60, - 0x22, 0x8f, 0x6b, 0xa0, 0x79, 0x28, 0x6c, 0x1f, 0xb2, 0xd3, 0x54, 0xbd, 0xc2, 0xf6, 0x61, 0x3e, - 0x75, 0x5b, 0xa5, 0xee, 0xfe, 0x64, 0xc3, 0xac, 0x47, 0xbe, 0x19, 0x91, 0x24, 0xe5, 0xe1, 0xd9, - 0xa7, 0x1e, 0x5e, 0x30, 0xd0, 0x1d, 0xa8, 0x7a, 0xc7, 0xdb, 0x07, 0x24, 0xbd, 0x08, 0x7b, 0x22, - 0x05, 0x59, 0x28, 0xce, 0xf4, 0x32, 0x39, 0x6a, 0x43, 0x7d, 0x3b, 0x26, 0x38, 0xe5, 0x5d, 0x49, - 0x58, 0x2a, 0xb5, 0xcd, 0x15, 0xa1, 0xaf, 0x8b, 0x44, 0x00, 0x51, 0x1c, 0xc3, 0x8a, 0x7a, 0x69, - 0x93, 0x01, 0x51, 0x5e, 0x6c, 0xc3, 0x8b, 0x2e, 0xca, 0x79, 0xd1, 0x45, 0xe8, 0x53, 0xa8, 0xee, - 0x12, 0x1c, 0xa7, 0x1d, 0x82, 0x39, 0xc2, 0x6a, 0x9b, 0xff, 0x13, 0x2e, 0x14, 0xdf, 0xb4, 0xcf, - 0xf4, 0xa9, 0xf1, 0x63, 0x92, 0x8a, 0xf8, 0x65, 0xc3, 0x58, 0xf1, 0x73, 0xc6, 0x8a, 0x4f, 0x8d, - 0x45, 0x7f, 0x71, 0xaf, 0x31, 0x6b, 0x18, 0x2b, 0x7e, 0xce, 0x58, 0xf1, 0xd1, 0x7d, 0x98, 0x3d, - 0xc6, 0xa3, 0x84, 0x6c, 0x1f, 0x36, 0x2a, 0xcc, 0x74, 0x59, 0xc2, 0x92, 0x73, 0x4d, 0x43, 0xa9, - 0x8b, 0x1e, 0xc3, 0xdc, 0x63, 0x92, 0x52, 0xb1, 0x00, 0x64, 0x95, 0x19, 0xff, 0x3f, 0x4b, 0x3a, - 0x93, 0x99, 0x2e, 0x4c, 0x3b, 0xf7, 0x57, 0x9b, 0xa2, 0x3a, 0x89, 0xc2, 0x20, 0x21, 0x6f, 0x13, - 0x1a, 0x4b, 0x50, 0xda, 0x89, 0xe3, 0x30, 0x66, 0x98, 0xa8, 0x7b, 0x9c, 0x40, 0x3b, 0x39, 0xc0, - 0xd8, 0x46, 0xd6, 0x26, 0x60, 0x78, 0x4e, 0x53, 0x11, 0xb3, 0x93, 0x43, 0x4c, 0xc9, 0x70, 0x63, - 0x22, 0xc6, 0x74, 0x63, 0x40, 0xe6, 0xa1, 0x0e, 0x19, 0xde, 0xf5, 0xc6, 0x24, 0x64, 0x0c, 0x07, - 0x1a, 0x66, 0x1e, 0xea, 0x98, 0x99, 0x35, 0xac, 0x35, 0xcc, 0x98, 0xd6, 0x19, 0x68, 0x1e, 0xea, - 0xa0, 0xa9, 0x18, 0xd6, 0x1a, 0x68, 0x4c, 0xeb, 0x0c, 0x35, 0x1f, 0x67, 0xa8, 0xe1, 0x8d, 0xbf, - 0x9e, 0x47, 0x8d, 0x61, 0xa9, 0x60, 0xb3, 0x9b, 0x87, 0x0d, 0x30, 0xeb, 0x1b, 0xd3, 0x61, 0x63, - 0xf8, 0xc8, 0xe3, 0xc6, 0x82, 0xca, 0x51, 0x44, 0x62, 0x9c, 0x86, 0x31, 0x7a, 0x0f, 0xec, 0xd3, - 0x71, 0x44, 0x72, 0x2b, 0xe6, 0x28, 0xa2, 0x4c, 0x8f, 0x89, 0xd0, 0x03, 0x7d, 0x7e, 0x33, 0xf4, - 0xd4, 0x36, 0x17, 0x85, 0x62, 0x26, 0x10, 0xb1, 0xf4, 0x51, 0x7f, 0x8f, 0x4e, 0x2e, 0x16, 0x54, - 0x8c, 0x97, 0x2b, 0x86, 0xae, 0xd4, 0xd4, 0xf7, 0x83, 0x6d, 0xec, 0x07, 0xf7, 0x2b, 0xb8, 0x36, - 0x65, 0x2e, 0xd1, 0x51, 0xaa, 0x20, 0x5f, 0xd8, 0x6b, 0xa3, 0x07, 0x50, 0x91, 0x2b, 0x54, 0x24, - 0xbb, 0xac, 0x87, 0x55, 0xfb, 0x55, 0x4e, 0x7b, 0x49, 0xbb, 0xd7, 0x61, 0x69, 0x1a, 0x8c, 0xdd, - 0x5b, 0x70, 0x6d, 0xca, 0x24, 0xcb, 0xc7, 0xa5, 0xe6, 0xd3, 0xe0, 0xeb, 0x9e, 0x80, 0x93, 0x9f, - 0x62, 0x62, 0xfc, 0x5b, 0x6a, 0xfc, 0x6f, 0x40, 0x59, 0xa0, 0xb1, 0xc0, 0xb6, 0xd3, 0x2b, 0xcb, - 0x2b, 0xd4, 0xdc, 0x5d, 0x58, 0x9c, 0xc0, 0x39, 0xba, 0x07, 0x55, 0xd9, 0xd7, 0xa4, 0x61, 0x31, - 0x47, 0x0b, 0xaa, 0xa1, 0x9c, 0x2f, 0xf1, 0xa8, 0xf4, 0xdc, 0x2f, 0xc1, 0xc9, 0xcf, 0xc9, 0xb7, - 0x57, 0xd2, 0x36, 0x2c, 0x4e, 0xfc, 0xa0, 0xb4, 0xc3, 0x5a, 0x6f, 0x76, 0xd8, 0x3f, 0x2c, 0x70, - 0xf2, 0xe3, 0x18, 0xdd, 0x85, 0x12, 0x07, 0xa4, 0x75, 0x35, 0x20, 0xb9, 0x96, 0xa8, 0x78, 0x41, - 0x55, 0xfc, 0x3a, 0x94, 0xc5, 0x38, 0xe4, 0x57, 0x22, 0x41, 0xa1, 0x0f, 0xa0, 0x74, 0x8c, 0x63, - 0x3c, 0x14, 0xf3, 0xcd, 0x51, 0x3b, 0x1c, 0xf7, 0x18, 0x5f, 0x7a, 0x65, 0x04, 0xda, 0x84, 0x32, - 0x95, 0x6c, 0xc9, 0xb5, 0xb5, 0xd4, 0xca, 0xee, 0x7c, 0xa7, 0xf2, 0x4b, 0x9e, 0x86, 0x6b, 0xba, - 0x77, 0x61, 0x71, 0x62, 0x4c, 0x50, 0xd4, 0x1f, 0xe3, 0xf1, 0x20, 0xc4, 0xfc, 0x3c, 0x75, 0x4f, - 0x92, 0x6e, 0x13, 0xe6, 0xcd, 0x7d, 0xa2, 0x75, 0xa7, 0xca, 0x80, 0xb7, 0x08, 0x0b, 0xb9, 0xd9, - 0xe1, 0xde, 0x86, 0xa5, 0x69, 0x7b, 0x64, 0xc2, 0xf4, 0x10, 0x96, 0xa7, 0x0e, 0x0e, 0xe3, 0xc2, - 0x64, 0xbd, 0xf9, 0x85, 0xe9, 0xf7, 0x12, 0x5d, 0x43, 0xa2, 0x54, 0x6c, 0x41, 0xc6, 0x61, 0x97, - 0x24, 0x89, 0x68, 0xd2, 0x72, 0x4b, 0xdd, 0x80, 0x85, 0x60, 0x2f, 0x38, 0x0f, 0xd5, 0xa4, 0xe3, - 0x2c, 0xf4, 0x21, 0x54, 0x4e, 0x2f, 0x03, 0xd6, 0x48, 0x81, 0x36, 0x89, 0x62, 0xc9, 0x96, 0x71, - 0x25, 0x4d, 0x47, 0x14, 0xbd, 0x57, 0x25, 0xbc, 0x75, 0x45, 0x03, 0x11, 0x99, 0x40, 0x8e, 0xa8, - 0x8c, 0x83, 0x3e, 0x82, 0xea, 0x89, 0xff, 0x3d, 0x99, 0xd6, 0x72, 0xc5, 0x57, 0x33, 0x5c, 0x32, - 0xd0, 0x27, 0x50, 0xf3, 0x70, 0xd0, 0x27, 0x22, 0x1e, 0xef, 0x3d, 0x92, 0x50, 0xc9, 0x24, 0xc2, - 0x52, 0x57, 0x46, 0x11, 0xbc, 0x43, 0x4b, 0x1e, 0x0e, 0x46, 0x43, 0xf9, 0x33, 0x39, 0xe9, 0xe2, - 0x80, 0x16, 0x82, 0x7b, 0xe3, 0xdb, 0xec, 0xa6, 0x36, 0xd7, 0x5f, 0xa9, 0x2b, 0xfc, 0x5f, 0xed, - 0x10, 0x7d, 0x0e, 0x8b, 0x07, 0x24, 0xee, 0x93, 0xa3, 0xce, 0x53, 0xd2, 0x95, 0x35, 0x32, 0xb7, - 0xde, 0x84, 0x5c, 0x78, 0x9e, 0x34, 0xa4, 0xdd, 0xd9, 0x27, 0x63, 0xee, 0xa4, 0x62, 0x74, 0x47, - 0xb2, 0x65, 0x77, 0x24, 0xcd, 0xca, 0x45, 0x70, 0x8f, 0xc4, 0xdc, 0xaa, 0x6a, 0x96, 0x2b, 0x93, - 0xa8, 0x72, 0x65, 0x2c, 0x5a, 0xae, 0xe3, 0xd8, 0x1f, 0xe2, 0x78, 0xbc, 0x4f, 0xc6, 0xc9, 0x01, - 0x1e, 0x3f, 0x22, 0x07, 0x61, 0xcf, 0x3f, 0xf7, 0x09, 0x07, 0x99, 0x58, 0x83, 0xb2, 0x5c, 0x57, - 0xea, 0xca, 0x72, 0x5d, 0xa9, 0xe4, 0xfe, 0x68, 0x65, 0xf8, 0x33, 0x1f, 0x48, 0x56, 0xfe, 0x81, - 0xb4, 0x0a, 0xd0, 0xc6, 0x29, 0xee, 0xe0, 0x84, 0xa8, 0x47, 0x8b, 0xc6, 0x41, 0x2e, 0xd4, 0x25, - 0x75, 0x88, 0x87, 0x44, 0xdc, 0xf7, 0x0d, 0x1e, 0x8d, 0xc0, 0x42, 0x31, 0x05, 0x9b, 0x29, 0x64, - 0x0c, 0xb7, 0xa9, 0x03, 0x1b, 0x21, 0xb0, 0x93, 0x71, 0xd0, 0x65, 0x89, 0x54, 0x3c, 0xf6, 0xed, - 0xde, 0xd1, 0x10, 0x4c, 0x13, 0xea, 0x52, 0x20, 0x04, 0xcc, 0x1b, 0xff, 0x9d, 0x6b, 0x1c, 0xf7, - 0x6b, 0x03, 0xb8, 0xa8, 0x09, 0xa5, 0x9d, 0xcb, 0x48, 0x2d, 0x0b, 0x68, 0xb1, 0x07, 0x29, 0x65, - 0x79, 0x5c, 0x80, 0x5a, 0x80, 0xce, 0x82, 0x6e, 0x38, 0x1c, 0xfa, 0x69, 0x4a, 0x7a, 0x02, 0x08, - 0xec, 0xa4, 0x75, 0x6f, 0x8a, 0xc4, 0xfd, 0xec, 0x35, 0xe8, 0x7e, 0x6d, 0x86, 0x37, 0x32, 0x78, - 0x21, 0x07, 0x8a, 0xfb, 0x64, 0x2c, 0x06, 0x22, 0xfd, 0x74, 0x7f, 0xb6, 0x0c, 0x28, 0x51, 0x8d, - 0x60, 0x34, 0x64, 0x1a, 0x25, 0x8f, 0x7e, 0xa2, 0x9b, 0x60, 0x93, 0xcb, 0x28, 0x16, 0x83, 0x43, - 0x3b, 0x91, 0x68, 0x3e, 0x93, 0xd2, 0xe9, 0x1f, 0xb3, 0x3a, 0x88, 0x3b, 0xae, 0xa0, 0xe8, 0x5b, - 0x3a, 0x8c, 0x7b, 0x24, 0x26, 0x3d, 0x9a, 0x35, 0x6b, 0x47, 0xc5, 0xd3, 0x59, 0xb4, 0x5d, 0xe9, - 0x65, 0x70, 0x74, 0x7e, 0x9e, 0x10, 0x3e, 0xf4, 0x4b, 0x5e, 0xc6, 0x70, 0xf1, 0x6b, 0xd0, 0x4a, - 0x3b, 0x78, 0x1e, 0x87, 0x43, 0x71, 0x26, 0xf6, 0x4d, 0x87, 0x72, 0x1a, 0x8a, 0x9a, 0x16, 0xd2, - 0x90, 0x86, 0x78, 0x46, 0xc6, 0x4f, 0x48, 0x37, 0x55, 0x77, 0xf0, 0x8c, 0xe1, 0x9e, 0x4d, 0xf9, - 0x35, 0xd3, 0xf7, 0x77, 0xd8, 0x79, 0x9a, 0x50, 0xa4, 0xb0, 0x5e, 0xd6, 0x3d, 0x45, 0xa3, 0x9b, - 0x30, 0x97, 0xe2, 0xb8, 0x4f, 0xd2, 0xa3, 0xce, 0x53, 0x8a, 0x14, 0xb1, 0xf0, 0x4c, 0xe6, 0xfa, - 0x1d, 0xf9, 0xc4, 0x46, 0x15, 0xb0, 0x0f, 0xc3, 0x80, 0x38, 0x33, 0x68, 0x0e, 0xaa, 0xc7, 0x38, - 0x4e, 0xfd, 0xd4, 0x0f, 0x03, 0xc7, 0xa2, 0x82, 0x5d, 0x9c, 0x5c, 0x38, 0x85, 0xf5, 0x7d, 0x28, - 0xf3, 0x1b, 0x22, 0x9a, 0x07, 0xd8, 0xea, 0xc9, 0x75, 0xe0, 0xcc, 0xa0, 0x45, 0x98, 0xe3, 0x97, - 0x20, 0xc9, 0xb2, 0xa8, 0x17, 0xce, 0xda, 0x1a, 0x0c, 0x9c, 0x02, 0x5a, 0x80, 0x1a, 0xbf, 0x65, - 0x31, 0x90, 0x3b, 0xc5, 0xf5, 0xdb, 0x30, 0xbb, 0x7d, 0xc8, 0x5f, 0xc5, 0x65, 0x28, 0x9c, 0x45, - 0xce, 0x0c, 0xaa, 0xd2, 0x25, 0x3c, 0x4a, 0x08, 0x0f, 0xda, 0x0e, 0xbf, 0x0b, 0x9c, 0xc2, 0xfa, - 0x19, 0xd4, 0xf5, 0x97, 0x34, 0x0b, 0x3d, 0x18, 0x84, 0x5d, 0x9c, 0xfa, 0x41, 0x9f, 0x67, 0x2b, - 0x68, 0xd2, 0x73, 0x2c, 0x54, 0x83, 0x59, 0x6f, 0x14, 0x04, 0x54, 0x56, 0x40, 0x00, 0xe5, 0x83, - 0xf0, 0x5b, 0xfa, 0x5d, 0xa4, 0x7a, 0xa7, 0xe1, 0xb0, 0x93, 0xa4, 0xf4, 0x90, 0xf6, 0xfa, 0x58, - 0x5e, 0x04, 0xa8, 0x40, 0xdd, 0xa9, 0x9c, 0x19, 0xe4, 0x98, 0x0f, 0x1e, 0xc7, 0xa2, 0x1c, 0xfd, - 0x86, 0xe7, 0x14, 0xa8, 0x89, 0xba, 0xdf, 0x70, 0xd7, 0x6a, 0xb5, 0x3b, 0x36, 0x4d, 0x41, 0x2c, - 0x66, 0xa7, 0x44, 0x2b, 0x63, 0xac, 0x5a, 0xa7, 0xfc, 0xa8, 0xfd, 0xe2, 0x9f, 0x55, 0xeb, 0xf9, - 0xcb, 0x55, 0xeb, 0xc5, 0xcb, 0x55, 0xeb, 0xef, 0x97, 0xab, 0x33, 0xbf, 0xfc, 0xbb, 0x6a, 0x7d, - 0xd1, 0xd2, 0xfe, 0x65, 0x1a, 0xe2, 0x34, 0xf6, 0x2f, 0xc3, 0xd8, 0xef, 0xfb, 0x81, 0x24, 0x02, - 0xb2, 0x11, 0x3d, 0xeb, 0x6f, 0x44, 0x9d, 0x0d, 0x36, 0xf2, 0x3a, 0x65, 0xf6, 0x07, 0xd2, 0xbd, - 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xe3, 0x8d, 0x4a, 0xb2, 0x12, 0x00, 0x00, + // 1820 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x49, + 0x15, 0xf7, 0xc8, 0x23, 0x59, 0x7a, 0x92, 0xed, 0x71, 0xc7, 0xf1, 0xaa, 0x42, 0xd6, 0x88, 0xa9, + 0xec, 0x62, 0x1c, 0x62, 0x17, 0x0e, 0xcb, 0x56, 0x41, 0xf8, 0xe3, 0x48, 0xa9, 0xc4, 0x15, 0xac, + 0xb8, 0xc6, 0xce, 0x1e, 0xd8, 0x2a, 0xa8, 0x96, 0xd4, 0x51, 0x26, 0x3b, 0x9a, 0x1e, 0x66, 0x5a, + 0x60, 0x71, 0xe3, 0xca, 0x17, 0xe0, 0xc2, 0x81, 0x0f, 0x01, 0x55, 0x7c, 0x84, 0xad, 0xe2, 0xb2, + 0x37, 0x6e, 0x14, 0x84, 0x2f, 0x42, 0xf5, 0xdf, 0xe9, 0x1e, 0x29, 0xce, 0x1e, 0xf6, 0xd6, 0xef, + 0x6f, 0xbf, 0x7e, 0xfd, 0xeb, 0xf7, 0xde, 0x0c, 0xb4, 0x8b, 0xd7, 0x38, 0x9f, 0x1c, 0x65, 0x39, + 0x65, 0x14, 0xd5, 0x05, 0x71, 0xe7, 0xc1, 0x34, 0x66, 0xaf, 0xe7, 0xa3, 0xa3, 0x31, 0x9d, 0x1d, + 0x4f, 0xe9, 0x94, 0x1e, 0x0b, 0xe9, 0x68, 0xfe, 0x4a, 0x50, 0x82, 0x10, 0x2b, 0x69, 0x75, 0x67, + 0x9b, 0xc5, 0x33, 0x52, 0x30, 0x3c, 0xcb, 0x14, 0x63, 0x2b, 0x8b, 0x33, 0x92, 0xc4, 0x29, 0x51, + 0x34, 0x64, 0x09, 0x4e, 0xe5, 0x3a, 0xfc, 0x97, 0x07, 0x5b, 0x97, 0x7c, 0x97, 0xe2, 0x9c, 0x30, + 0x3c, 0xc1, 0x0c, 0xa3, 0x8f, 0xa0, 0x71, 0x41, 0x93, 0x78, 0xbc, 0xe8, 0x7a, 0x3d, 0xef, 0x60, + 0xeb, 0x64, 0xf3, 0x48, 0xc6, 0x24, 0x99, 0x91, 0x12, 0xa2, 0x1e, 0xb4, 0xa5, 0x61, 0x9f, 0xce, + 0x53, 0xd6, 0xad, 0xf5, 0xbc, 0x83, 0xcd, 0xc8, 0x66, 0xa1, 0x2e, 0x6c, 0x7c, 0x46, 0xf2, 0x22, + 0xa6, 0x69, 0x77, 0x5d, 0x48, 0x35, 0x89, 0xee, 0x42, 0xeb, 0x74, 0x3c, 0xe6, 0x4a, 0x67, 0x83, + 0xae, 0xdf, 0xf3, 0x0e, 0xfc, 0xa8, 0x64, 0xa0, 0x03, 0xd8, 0x3e, 0xc7, 0xd7, 0x11, 0xc9, 0x92, + 0x78, 0x8c, 0xa5, 0xf7, 0xba, 0xb0, 0xaf, 0xb2, 0xd1, 0x1d, 0x68, 0x8a, 0x0d, 0xcf, 0x06, 0x45, + 0xb7, 0xd1, 0x5b, 0x3f, 0xf0, 0x23, 0x43, 0x87, 0xff, 0xf0, 0x00, 0xae, 0xf0, 0x28, 0x21, 0x82, + 0xc3, 0x83, 0x11, 0xd4, 0xd9, 0x40, 0x1c, 0xcb, 0x8f, 0x34, 0xc9, 0x25, 0xca, 0x48, 0x1c, 0xc2, + 0x8f, 0x34, 0x69, 0x65, 0x62, 0xfd, 0xa6, 0x4c, 0x58, 0xe7, 0xf4, 0xdd, 0x73, 0x7e, 0x02, 0x4d, + 0x15, 0x6f, 0xd1, 0xad, 0xf7, 0xd6, 0x0f, 0xda, 0x27, 0xb7, 0x94, 0x0b, 0xb1, 0x85, 0x92, 0x3d, + 0xf6, 0xbf, 0xfc, 0xf7, 0xb7, 0xd7, 0x22, 0xa3, 0x1a, 0xfe, 0xd1, 0x83, 0x8e, 0xad, 0xc0, 0xf3, + 0xa5, 0x96, 0x26, 0xfc, 0x92, 0x81, 0xbe, 0x07, 0xf5, 0x4b, 0x86, 0x19, 0x11, 0xe1, 0x6f, 0x99, + 0x2d, 0x94, 0x82, 0x10, 0x45, 0x52, 0x03, 0x6d, 0x41, 0xad, 0x3f, 0x14, 0xa7, 0x69, 0x45, 0xb5, + 0xfe, 0xb0, 0x1a, 0xba, 0x6f, 0x42, 0x0f, 0xff, 0xec, 0xc3, 0x46, 0x44, 0x7e, 0x3b, 0x27, 0x05, + 0x93, 0xdb, 0x8b, 0xa5, 0xbd, 0xbd, 0x62, 0xa0, 0xfb, 0xd0, 0x8a, 0x2e, 0xfa, 0xe7, 0x84, 0xbd, + 0xa6, 0x13, 0x15, 0x82, 0x4e, 0x94, 0x64, 0x46, 0xa5, 0x1c, 0x0d, 0xa0, 0xd3, 0xcf, 0x09, 0x66, + 0xf2, 0x56, 0x0a, 0x11, 0x4a, 0xfb, 0xe4, 0x8e, 0xd2, 0xb7, 0x45, 0x6a, 0x03, 0x95, 0x1c, 0xc7, + 0x8a, 0x7b, 0x19, 0x90, 0x84, 0x18, 0x2f, 0xbe, 0xe3, 0xc5, 0x16, 0x55, 0xbc, 0xd8, 0x22, 0xf4, + 0x13, 0x68, 0x3d, 0x23, 0x38, 0x67, 0x23, 0x82, 0x25, 0xc2, 0xda, 0x27, 0x1f, 0x28, 0x17, 0x86, + 0xef, 0xda, 0x97, 0xfa, 0xdc, 0xf8, 0x29, 0x61, 0x6a, 0xff, 0x86, 0x63, 0x6c, 0xf8, 0x15, 0x63, + 0xc3, 0xe7, 0xc6, 0xea, 0x7e, 0xf1, 0xa4, 0xbb, 0xe1, 0x18, 0x1b, 0x7e, 0xc5, 0xd8, 0xf0, 0xd1, + 0x27, 0xb0, 0x71, 0x81, 0xe7, 0x05, 0xe9, 0x0f, 0xbb, 0x4d, 0x61, 0x7a, 0x5b, 0xc3, 0x52, 0x72, + 0x5d, 0x43, 0xad, 0x8b, 0x9e, 0xc2, 0xe6, 0x53, 0xc2, 0xb8, 0x58, 0x01, 0xb2, 0x25, 0x8c, 0xbf, + 0x55, 0x06, 0x5d, 0xca, 0x5c, 0x17, 0xae, 0x5d, 0xf8, 0x77, 0x9f, 0xa3, 0xba, 0xc8, 0x68, 0x5a, + 0x90, 0x6f, 0x12, 0x1a, 0xbb, 0x50, 0x7f, 0x92, 0xe7, 0x34, 0x17, 0x98, 0xe8, 0x44, 0x92, 0x40, + 0x4f, 0x2a, 0x80, 0xf1, 0x9d, 0xa8, 0x5d, 0xc0, 0xc8, 0x98, 0x56, 0x22, 0xe6, 0x49, 0x05, 0x31, + 0x75, 0xc7, 0x8d, 0x8b, 0x18, 0xd7, 0x8d, 0x03, 0x99, 0x47, 0x36, 0x64, 0xe4, 0xad, 0x77, 0x97, + 0x21, 0xe3, 0x38, 0xb0, 0x30, 0xf3, 0xc8, 0xc6, 0xcc, 0x86, 0x63, 0x6d, 0x61, 0xc6, 0xb5, 0x2e, + 0x41, 0xf3, 0xc8, 0x06, 0x4d, 0xd3, 0xb1, 0xb6, 0x40, 0xe3, 0x5a, 0x97, 0xa8, 0xf9, 0x51, 0x89, + 0x1a, 0x79, 0xf1, 0x7b, 0x55, 0xd4, 0x38, 0x96, 0x06, 0x36, 0xcf, 0xaa, 0xb0, 0x01, 0x61, 0x7d, + 0x77, 0x35, 0x6c, 0x1c, 0x1f, 0x55, 0xdc, 0x78, 0xd0, 0x7c, 0x91, 0x91, 0x1c, 0x33, 0x9a, 0xa3, + 0xef, 0x80, 0x7f, 0xb5, 0xc8, 0x48, 0xa5, 0xc5, 0xbc, 0xc8, 0x38, 0x33, 0x12, 0x22, 0xf4, 0xa9, + 0x5d, 0xbf, 0x05, 0x7a, 0xda, 0x27, 0x3b, 0x4a, 0xb1, 0x14, 0xa8, 0xbd, 0xec, 0x52, 0xff, 0x90, + 0x57, 0x2e, 0xb1, 0xa9, 0x2a, 0x2f, 0x37, 0x14, 0x5d, 0xad, 0x69, 0xf7, 0x07, 0xdf, 0xe9, 0x0f, + 0xe1, 0xaf, 0xe1, 0xd6, 0x8a, 0xba, 0xc4, 0x4b, 0xa9, 0x81, 0x7c, 0xed, 0x6c, 0x80, 0x3e, 0x85, + 0xa6, 0x6e, 0xa1, 0x2a, 0xd8, 0xdb, 0xf6, 0xb6, 0xa6, 0xbf, 0xea, 0x6a, 0xaf, 0xe9, 0x70, 0x0f, + 0x76, 0x57, 0xc1, 0x38, 0xfc, 0x08, 0x6e, 0xad, 0xa8, 0x64, 0xd5, 0x7d, 0xb9, 0xf9, 0x2a, 0xf8, + 0x86, 0x97, 0x10, 0x54, 0xab, 0x98, 0x2a, 0xff, 0x9e, 0x29, 0xff, 0xc7, 0xd0, 0x50, 0x68, 0xac, + 0x89, 0xee, 0xf4, 0xce, 0xf4, 0x2a, 0xb5, 0xf0, 0x19, 0xec, 0x2c, 0xe1, 0x1c, 0x3d, 0x84, 0x96, + 0xbe, 0xd7, 0xa2, 0xeb, 0x09, 0x47, 0xdb, 0xe6, 0x42, 0x25, 0x5f, 0xe3, 0xd1, 0xe8, 0x85, 0x9f, + 0x43, 0x50, 0xad, 0x93, 0xdf, 0x5c, 0x4a, 0x07, 0xb0, 0xb3, 0xf4, 0xa0, 0xac, 0xc3, 0x7a, 0x5f, + 0xef, 0xb0, 0xff, 0xf4, 0x20, 0xa8, 0x96, 0x63, 0xf4, 0x00, 0xea, 0x12, 0x90, 0xde, 0xcd, 0x80, + 0x94, 0x5a, 0x2a, 0xe3, 0x35, 0x93, 0xf1, 0x3d, 0x68, 0xa8, 0x72, 0x28, 0x47, 0x22, 0x45, 0xa1, + 0xef, 0x43, 0xfd, 0x02, 0xe7, 0x78, 0xa6, 0xea, 0x5b, 0x60, 0x7a, 0x38, 0x9e, 0x08, 0xbe, 0xf6, + 0x2a, 0x08, 0x74, 0x02, 0x0d, 0x2e, 0x39, 0xd5, 0x6d, 0x6b, 0xf7, 0xa8, 0x9c, 0xf9, 0xae, 0xf4, + 0x4a, 0x9f, 0x46, 0x6a, 0x86, 0x0f, 0x60, 0x67, 0xa9, 0x4c, 0x70, 0xd4, 0x5f, 0xe0, 0x45, 0x42, + 0xb1, 0x3c, 0x4f, 0x27, 0xd2, 0x64, 0xd8, 0x83, 0x2d, 0xb7, 0x9f, 0x58, 0xb7, 0xd3, 0x12, 0xc0, + 0xdb, 0x81, 0xed, 0x4a, 0xed, 0x08, 0x3f, 0x86, 0xdd, 0x55, 0x7d, 0x64, 0xc9, 0x74, 0x08, 0xb7, + 0x57, 0x16, 0x0e, 0x67, 0x60, 0xf2, 0xbe, 0xfe, 0xc0, 0xf4, 0xb7, 0x0d, 0xde, 0x86, 0x54, 0xaa, + 0x44, 0x83, 0xcc, 0xe9, 0x98, 0x14, 0x85, 0xba, 0xa4, 0xdb, 0x47, 0x66, 0x02, 0x56, 0x82, 0xb3, + 0xf4, 0x15, 0x35, 0x95, 0x4e, 0xb2, 0xd0, 0x0f, 0xa0, 0x79, 0x75, 0x9d, 0x8a, 0x8b, 0x54, 0x68, + 0xd3, 0x28, 0xd6, 0x6c, 0xbd, 0xaf, 0xa6, 0x79, 0x89, 0xe2, 0x73, 0x55, 0x21, 0xaf, 0x6e, 0xdd, + 0x41, 0x44, 0x29, 0xd0, 0x25, 0xaa, 0xe4, 0xa0, 0x1f, 0x42, 0xeb, 0x32, 0xfe, 0x03, 0x59, 0x75, + 0xe5, 0x86, 0x6f, 0x6a, 0xb8, 0x66, 0xa0, 0x1f, 0x43, 0x3b, 0xc2, 0xe9, 0x94, 0xa8, 0xfd, 0xe4, + 0xdd, 0x23, 0x0d, 0x95, 0x52, 0xa2, 0x2c, 0x6d, 0x65, 0x94, 0xc1, 0x87, 0x3c, 0xe5, 0x34, 0x99, + 0xcf, 0xf4, 0x33, 0xb9, 0x1c, 0xe3, 0x94, 0x27, 0x42, 0x7a, 0x93, 0xdd, 0xec, 0x9e, 0x55, 0xd7, + 0xdf, 0xa9, 0xab, 0xfc, 0xdf, 0xec, 0x10, 0xfd, 0x12, 0x76, 0xce, 0x49, 0x3e, 0x25, 0x2f, 0x46, + 0x6f, 0xc8, 0x58, 0xe7, 0xc8, 0xed, 0x7a, 0x4b, 0x72, 0xe5, 0x79, 0xd9, 0x90, 0xdf, 0xce, 0x73, + 0xb2, 0x90, 0x4e, 0x9a, 0xce, 0xed, 0x68, 0xb6, 0xbe, 0x1d, 0x4d, 0xa3, 0x33, 0x08, 0x38, 0x28, + 0x48, 0xfe, 0x78, 0x1e, 0x27, 0x12, 0x1b, 0xaa, 0xf7, 0x7d, 0x60, 0x3d, 0x2f, 0x5b, 0xac, 0x5c, + 0x2c, 0x99, 0xf1, 0xec, 0x5d, 0xe4, 0xf1, 0x0c, 0xe7, 0x8b, 0xe7, 0x64, 0x51, 0x9c, 0xe3, 0xc5, + 0x63, 0x72, 0x4e, 0x27, 0xf1, 0xab, 0x98, 0x28, 0xbf, 0xe0, 0x64, 0xef, 0x46, 0x5d, 0x9d, 0xbd, + 0x1b, 0x95, 0xd0, 0x2f, 0x60, 0x93, 0x47, 0x31, 0x24, 0xd7, 0x4c, 0xee, 0xd0, 0x56, 0x2f, 0xbd, + 0x8c, 0xdc, 0xc8, 0x74, 0xbf, 0x75, 0x98, 0xa8, 0x0f, 0x5b, 0x9c, 0xd1, 0x4f, 0x68, 0xa1, 0x80, + 0xd6, 0x71, 0x6a, 0xa8, 0x2b, 0x54, 0x3e, 0x2a, 0x26, 0xe8, 0x73, 0xd8, 0xeb, 0xd3, 0x24, 0x21, + 0x63, 0x76, 0x45, 0x67, 0xa3, 0x82, 0xd1, 0x54, 0xa3, 0x6f, 0x53, 0x38, 0xfb, 0x50, 0x0f, 0x62, + 0x2b, 0x95, 0x94, 0xd3, 0x77, 0xb8, 0x08, 0xff, 0xe4, 0x95, 0x4f, 0xce, 0xfd, 0x26, 0xf4, 0xaa, + 0xdf, 0x84, 0xfb, 0x00, 0x03, 0xcc, 0xf0, 0x08, 0x17, 0xc4, 0x7c, 0xa7, 0x59, 0x1c, 0x14, 0x42, + 0x47, 0x53, 0x43, 0x3c, 0x23, 0xea, 0x13, 0xc7, 0xe1, 0xf1, 0x1d, 0xc4, 0x56, 0x42, 0xc1, 0x17, + 0x0a, 0x25, 0x23, 0xec, 0xd9, 0x6f, 0x19, 0x21, 0xf0, 0x8b, 0x45, 0x3a, 0x16, 0x81, 0x34, 0x23, + 0xb1, 0x0e, 0xef, 0x5b, 0x8f, 0x96, 0x07, 0x34, 0xe6, 0xd8, 0x4f, 0x85, 0x37, 0x59, 0xda, 0x2c, + 0x4e, 0xf8, 0x1b, 0xe7, 0xad, 0xa2, 0x1e, 0xd4, 0x9f, 0x5c, 0x67, 0xa6, 0x3f, 0xc2, 0x91, 0xf8, + 0x06, 0xe7, 0xac, 0x48, 0x0a, 0xd0, 0x11, 0xa0, 0x97, 0xe9, 0x98, 0xce, 0x66, 0x31, 0x63, 0x64, + 0xa2, 0xb0, 0x2f, 0x4e, 0xda, 0x89, 0x56, 0x48, 0xc2, 0x9f, 0xbd, 0xeb, 0x66, 0xd0, 0x3d, 0xd8, + 0x1c, 0x4b, 0x89, 0xf5, 0x1d, 0xef, 0x47, 0x2e, 0x33, 0xfc, 0xf9, 0x7b, 0x0a, 0xc2, 0x7b, 0x4f, + 0x78, 0xb7, 0x7c, 0x91, 0x28, 0x80, 0xf5, 0xe7, 0x64, 0xa1, 0x7a, 0x08, 0x5f, 0x86, 0x7f, 0xf1, + 0x96, 0x5f, 0x1f, 0x6f, 0x37, 0x39, 0x49, 0xf8, 0xa5, 0xe8, 0x76, 0xa3, 0x48, 0xb4, 0x0f, 0x3e, + 0xb9, 0xce, 0x72, 0x55, 0x78, 0xed, 0xf4, 0x08, 0x3e, 0xff, 0xd2, 0x2f, 0xc6, 0x38, 0x15, 0x33, + 0x23, 0xbf, 0xdb, 0x7a, 0x64, 0x68, 0x74, 0x02, 0xbb, 0x4c, 0xa7, 0xe0, 0x34, 0xcb, 0x92, 0x85, + 0x3a, 0xb6, 0x2f, 0xf4, 0x56, 0xca, 0xc2, 0xef, 0xc2, 0x8e, 0x15, 0x5d, 0x44, 0x8a, 0x79, 0xc2, + 0xf8, 0xa5, 0xcf, 0xe7, 0xb1, 0x6e, 0x85, 0x62, 0x1d, 0xfe, 0xb4, 0xf2, 0x0e, 0x57, 0x29, 0xf1, + 0x73, 0xc9, 0xc4, 0xc8, 0x41, 0xaa, 0x15, 0x69, 0x32, 0x3c, 0x90, 0x8f, 0x90, 0x9b, 0xab, 0x4d, + 0xf6, 0xa0, 0x91, 0x8b, 0x95, 0xf2, 0xa0, 0xa8, 0xf0, 0x5e, 0xf5, 0xb9, 0xae, 0x0c, 0x07, 0xbf, + 0xa7, 0x10, 0x71, 0xa3, 0x57, 0x39, 0x9d, 0x69, 0x23, 0xbe, 0xe6, 0xed, 0x97, 0x51, 0x05, 0xa5, + 0x1a, 0xa3, 0xfc, 0x21, 0x7c, 0x41, 0x16, 0x9f, 0x91, 0x31, 0x33, 0x5f, 0x5b, 0x25, 0x23, 0x7c, + 0xb9, 0xa2, 0x6e, 0xf3, 0xfc, 0xd3, 0xd1, 0x9b, 0x82, 0x3f, 0x10, 0x01, 0xe1, 0x4e, 0x64, 0x68, + 0x8e, 0x37, 0x86, 0xf3, 0x29, 0x61, 0x2f, 0x46, 0x6f, 0xf8, 0x03, 0x51, 0xa3, 0x8d, 0xcb, 0x3c, + 0xbc, 0xaf, 0x7f, 0xa6, 0xa0, 0x26, 0xf8, 0x43, 0x9a, 0x92, 0x60, 0x0d, 0x6d, 0x42, 0xeb, 0x02, + 0xe7, 0x2c, 0x66, 0x31, 0x4d, 0x03, 0x8f, 0x0b, 0x9e, 0xe1, 0xe2, 0x75, 0x50, 0x3b, 0x7c, 0x0e, + 0x0d, 0xf9, 0x2d, 0x80, 0xb6, 0x00, 0x4e, 0x27, 0xba, 0xf1, 0x07, 0x6b, 0x68, 0x07, 0x36, 0xe5, + 0xb8, 0xab, 0x59, 0x1e, 0xf7, 0x22, 0x59, 0xa7, 0x49, 0x12, 0xd4, 0xd0, 0x36, 0xb4, 0xe5, 0x3c, + 0x2d, 0xde, 0x76, 0xb0, 0x7e, 0xf8, 0x31, 0x6c, 0xf4, 0x87, 0xf2, 0xff, 0x47, 0x03, 0x6a, 0x2f, + 0xb3, 0x60, 0x0d, 0xb5, 0xf8, 0xb8, 0x35, 0x2f, 0x88, 0xdc, 0x74, 0x40, 0x7f, 0x9f, 0x06, 0xb5, + 0xc3, 0x97, 0xd0, 0xb1, 0xff, 0x99, 0x88, 0xad, 0x93, 0x84, 0x8e, 0x31, 0x8b, 0xd3, 0xa9, 0x8c, + 0x56, 0xd1, 0x64, 0x12, 0x78, 0xa8, 0x0d, 0x1b, 0xd1, 0x3c, 0x4d, 0xb9, 0xac, 0x86, 0x00, 0x1a, + 0xe7, 0xf4, 0x77, 0x7c, 0xbd, 0xce, 0xf5, 0xcc, 0x93, 0x0c, 0xfc, 0xc3, 0x85, 0x1e, 0xf9, 0xb8, + 0xc0, 0x4c, 0xcf, 0xc1, 0x1a, 0x0a, 0xdc, 0x4f, 0xdb, 0xc0, 0xe3, 0x1c, 0x7b, 0x96, 0x0f, 0x6a, + 0xdc, 0xc4, 0x4c, 0xb2, 0xd2, 0xb5, 0x19, 0xe2, 0x02, 0x9f, 0x87, 0xa0, 0x46, 0xb0, 0xa0, 0xce, + 0x33, 0xe3, 0x0c, 0x55, 0x41, 0xe3, 0xf1, 0xe0, 0xab, 0xff, 0xee, 0x7b, 0x5f, 0xbe, 0xdd, 0xf7, + 0xbe, 0x7a, 0xbb, 0xef, 0xfd, 0xe7, 0xed, 0xfe, 0xda, 0x5f, 0xff, 0xb7, 0xef, 0xfd, 0xea, 0xc8, + 0xfa, 0x9f, 0x38, 0xc3, 0x2c, 0x8f, 0xaf, 0x69, 0x1e, 0x4f, 0xe3, 0x54, 0x13, 0x29, 0x39, 0xce, + 0xbe, 0x98, 0x1e, 0x67, 0xa3, 0x63, 0x51, 0xdb, 0x47, 0x0d, 0xf1, 0xab, 0xf0, 0xe1, 0xff, 0x03, + 0x00, 0x00, 0xff, 0xff, 0x47, 0xd8, 0xa1, 0xa8, 0x9c, 0x14, 0x00, 0x00, } func (m *ShardsMetadata) Marshal() (dAtA []byte, err error) { @@ -3257,6 +3530,36 @@ func (m *ReadParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + { + size, err := m.CollectTombstonesParam.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShard(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x6a + { + size, err := m.ReadCloseParam.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShard(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x62 + { + size, err := m.ReadNextParam.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShard(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a { size, err := m.PrimaryKeysMayBeModifiedParam.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -3268,7 +3571,7 @@ func (m *ReadParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x52 { - size, err := m.ReaderParam.MarshalToSizedBuffer(dAtA[:i]) + size, err := m.ReaderBuildParam.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -3530,6 +3833,38 @@ func (m *RangesParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *CollectTombstonesParam) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CollectTombstonesParam) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CollectTombstonesParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.CollectPolicy != 0 { + i = encodeVarintShard(dAtA, i, uint64(m.CollectPolicy)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *GetColumMetadataScanInfoParam) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -3598,7 +3933,7 @@ func (m *KeyParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *ReaderParam) Marshal() (dAtA []byte, err error) { +func (m *ReaderBuildParam) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -3608,12 +3943,12 @@ func (m *ReaderParam) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *ReaderParam) MarshalTo(dAtA []byte) (int, error) { +func (m *ReaderBuildParam) MarshalTo(dAtA []byte) (int, error) { size := m.ProtoSize() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *ReaderParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ReaderBuildParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -3622,47 +3957,39 @@ func (m *ReaderParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.TxnOffset != 0 { - i = encodeVarintShard(dAtA, i, uint64(m.TxnOffset)) - i-- - dAtA[i] = 0x28 - } - if m.OrderedScan { - i-- - if m.OrderedScan { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } + if m.TombstoneApplyPolicy != 0 { + i = encodeVarintShard(dAtA, i, uint64(m.TombstoneApplyPolicy)) i-- dAtA[i] = 0x20 } - if len(m.Ranges) > 0 { - i -= len(m.Ranges) - copy(dAtA[i:], m.Ranges) - i = encodeVarintShard(dAtA, i, uint64(len(m.Ranges))) + if m.ScanType != 0 { + i = encodeVarintShard(dAtA, i, uint64(m.ScanType)) i-- - dAtA[i] = 0x1a + dAtA[i] = 0x18 } - { - size, err := m.Expr.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if m.Expr != nil { + { + size, err := m.Expr.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShard(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintShard(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 - if m.Num != 0 { - i = encodeVarintShard(dAtA, i, uint64(m.Num)) + if len(m.RelData) > 0 { + i -= len(m.RelData) + copy(dAtA[i:], m.RelData) + i = encodeVarintShard(dAtA, i, uint64(len(m.RelData))) i-- - dAtA[i] = 0x8 + dAtA[i] = 0xa } return len(dAtA) - i, nil } -func (m *PrimaryKeysMayBeModifiedParam) Marshal() (dAtA []byte, err error) { +func (m *ReaderBuildResult) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -3672,12 +3999,12 @@ func (m *PrimaryKeysMayBeModifiedParam) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *PrimaryKeysMayBeModifiedParam) MarshalTo(dAtA []byte) (int, error) { +func (m *ReaderBuildResult) MarshalTo(dAtA []byte) (int, error) { size := m.ProtoSize() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *PrimaryKeysMayBeModifiedParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ReaderBuildResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -3686,8 +4013,153 @@ func (m *PrimaryKeysMayBeModifiedParam) MarshalToSizedBuffer(dAtA []byte) (int, i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.KeyVector) > 0 { - i -= len(m.KeyVector) + if len(m.Uuid) > 0 { + i -= len(m.Uuid) + copy(dAtA[i:], m.Uuid) + i = encodeVarintShard(dAtA, i, uint64(len(m.Uuid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ReadNextParam) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReadNextParam) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ReadNextParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Columns) > 0 { + for iNdEx := len(m.Columns) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Columns[iNdEx]) + copy(dAtA[i:], m.Columns[iNdEx]) + i = encodeVarintShard(dAtA, i, uint64(len(m.Columns[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Uuid) > 0 { + i -= len(m.Uuid) + copy(dAtA[i:], m.Uuid) + i = encodeVarintShard(dAtA, i, uint64(len(m.Uuid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ReadNextResult) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReadNextResult) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ReadNextResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Result) > 0 { + i -= len(m.Result) + copy(dAtA[i:], m.Result) + i = encodeVarintShard(dAtA, i, uint64(len(m.Result))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ReadCloseParam) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReadCloseParam) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ReadCloseParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Uuid) > 0 { + i -= len(m.Uuid) + copy(dAtA[i:], m.Uuid) + i = encodeVarintShard(dAtA, i, uint64(len(m.Uuid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PrimaryKeysMayBeModifiedParam) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PrimaryKeysMayBeModifiedParam) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PrimaryKeysMayBeModifiedParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.KeyVector) > 0 { + i -= len(m.KeyVector) copy(dAtA[i:], m.KeyVector) i = encodeVarintShard(dAtA, i, uint64(len(m.KeyVector))) i-- @@ -4197,10 +4669,16 @@ func (m *ReadParam) ProtoSize() (n int) { n += 1 + l + sovShard(uint64(l)) l = m.KeyParam.ProtoSize() n += 1 + l + sovShard(uint64(l)) - l = m.ReaderParam.ProtoSize() + l = m.ReaderBuildParam.ProtoSize() n += 1 + l + sovShard(uint64(l)) l = m.PrimaryKeysMayBeModifiedParam.ProtoSize() n += 1 + l + sovShard(uint64(l)) + l = m.ReadNextParam.ProtoSize() + n += 1 + l + sovShard(uint64(l)) + l = m.ReadCloseParam.ProtoSize() + n += 1 + l + sovShard(uint64(l)) + l = m.CollectTombstonesParam.ProtoSize() + n += 1 + l + sovShard(uint64(l)) if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4286,6 +4764,21 @@ func (m *RangesParam) ProtoSize() (n int) { return n } +func (m *CollectTombstonesParam) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.CollectPolicy != 0 { + n += 1 + sovShard(uint64(m.CollectPolicy)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *GetColumMetadataScanInfoParam) ProtoSize() (n int) { if m == nil { return 0 @@ -4318,26 +4811,95 @@ func (m *KeyParam) ProtoSize() (n int) { return n } -func (m *ReaderParam) ProtoSize() (n int) { +func (m *ReaderBuildParam) ProtoSize() (n int) { if m == nil { return 0 } var l int _ = l - if m.Num != 0 { - n += 1 + sovShard(uint64(m.Num)) + l = len(m.RelData) + if l > 0 { + n += 1 + l + sovShard(uint64(l)) } - l = m.Expr.ProtoSize() - n += 1 + l + sovShard(uint64(l)) - l = len(m.Ranges) + if m.Expr != nil { + l = m.Expr.ProtoSize() + n += 1 + l + sovShard(uint64(l)) + } + if m.ScanType != 0 { + n += 1 + sovShard(uint64(m.ScanType)) + } + if m.TombstoneApplyPolicy != 0 { + n += 1 + sovShard(uint64(m.TombstoneApplyPolicy)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ReaderBuildResult) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Uuid) if l > 0 { n += 1 + l + sovShard(uint64(l)) } - if m.OrderedScan { - n += 2 + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ReadNextParam) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Uuid) + if l > 0 { + n += 1 + l + sovShard(uint64(l)) + } + if len(m.Columns) > 0 { + for _, s := range m.Columns { + l = len(s) + n += 1 + l + sovShard(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ReadNextResult) ProtoSize() (n int) { + if m == nil { + return 0 } - if m.TxnOffset != 0 { - n += 1 + sovShard(uint64(m.TxnOffset)) + var l int + _ = l + l = len(m.Result) + if l > 0 { + n += 1 + l + sovShard(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ReadCloseParam) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Uuid) + if l > 0 { + n += 1 + l + sovShard(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -7296,7 +7858,7 @@ func (m *ReadParam) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 9: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ReaderParam", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ReaderBuildParam", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7323,7 +7885,7 @@ func (m *ReadParam) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if err := m.ReaderParam.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ReaderBuildParam.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -7360,62 +7922,11 @@ func (m *ReadParam) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShard(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShard - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *TxnTable) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShard - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: TxnTable: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: TxnTable: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field AccountID", wireType) + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadNextParam", wireType) } - m.AccountID = 0 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShard @@ -7425,14 +7936,164 @@ func (m *TxnTable) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.AccountID |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field DatabaseID", wireType) + if msglen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.ReadNextParam.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadCloseParam", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.ReadCloseParam.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 13: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CollectTombstonesParam", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.CollectTombstonesParam.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TxnTable) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TxnTable: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TxnTable: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountID", wireType) + } + m.AccountID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DatabaseID", wireType) } m.DatabaseID = 0 for shift := uint(0); ; shift += 7 { @@ -7808,6 +8469,76 @@ func (m *RangesParam) Unmarshal(dAtA []byte) error { } return nil } +func (m *CollectTombstonesParam) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CollectTombstonesParam: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CollectTombstonesParam: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CollectPolicy", wireType) + } + m.CollectPolicy = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CollectPolicy |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GetColumMetadataScanInfoParam) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7976,7 +8707,7 @@ func (m *KeyParam) Unmarshal(dAtA []byte) error { } return nil } -func (m *ReaderParam) Unmarshal(dAtA []byte) error { +func (m *ReaderBuildParam) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -7999,17 +8730,17 @@ func (m *ReaderParam) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ReaderParam: wiretype end group for non-group") + return fmt.Errorf("proto: ReaderBuildParam: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ReaderParam: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ReaderBuildParam: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Num", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RelData", wireType) } - m.Num = 0 + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShard @@ -8019,11 +8750,26 @@ func (m *ReaderParam) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Num |= int32(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RelData = append(m.RelData[:0], dAtA[iNdEx:postIndex]...) + if m.RelData == nil { + m.RelData = []byte{} + } + iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Expr", wireType) @@ -8053,15 +8799,18 @@ func (m *ReaderParam) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + if m.Expr == nil { + m.Expr = &plan.Expr{} + } if err := m.Expr.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ranges", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ScanType", wireType) } - var byteLen int + m.ScanType = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShard @@ -8071,31 +8820,16 @@ func (m *ReaderParam) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.ScanType |= int32(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return ErrInvalidLengthShard - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShard - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Ranges = append(m.Ranges[:0], dAtA[iNdEx:postIndex]...) - if m.Ranges == nil { - m.Ranges = []byte{} - } - iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field OrderedScan", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TombstoneApplyPolicy", wireType) } - var v int + m.TombstoneApplyPolicy = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShard @@ -8105,31 +8839,383 @@ func (m *ReaderParam) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + m.TombstoneApplyPolicy |= int32(b&0x7F) << shift if b < 0x80 { break } } - m.OrderedScan = bool(v != 0) - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TxnOffset", wireType) + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err } - m.TxnOffset = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShard - } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReaderBuildResult) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReaderBuildResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReaderBuildResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Uuid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Uuid = append(m.Uuid[:0], dAtA[iNdEx:postIndex]...) + if m.Uuid == nil { + m.Uuid = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReadNextParam) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReadNextParam: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReadNextParam: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Uuid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ - m.TxnOffset |= int32(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Uuid = append(m.Uuid[:0], dAtA[iNdEx:postIndex]...) + if m.Uuid == nil { + m.Uuid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Columns = append(m.Columns, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReadNextResult) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReadNextResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReadNextResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Result", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Result = append(m.Result[:0], dAtA[iNdEx:postIndex]...) + if m.Result == nil { + m.Result = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShard(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShard + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReadCloseParam) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReadCloseParam: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReadCloseParam: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Uuid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShard + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShard + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShard + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Uuid = append(m.Uuid[:0], dAtA[iNdEx:postIndex]...) + if m.Uuid == nil { + m.Uuid = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipShard(dAtA[iNdEx:]) diff --git a/pkg/shardservice/service.go b/pkg/shardservice/service.go index 0cc4c328555a..0dc4277143d0 100644 --- a/pkg/shardservice/service.go +++ b/pkg/shardservice/service.go @@ -23,6 +23,8 @@ import ( "time" "github.com/RoaringBitmap/roaring/roaring64" + "go.uber.org/zap" + "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/common/log" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -33,7 +35,6 @@ import ( pb "github.com/matrixorigin/matrixone/pkg/pb/shard" "github.com/matrixorigin/matrixone/pkg/txn/client" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" - "go.uber.org/zap" ) type Option func(*service) diff --git a/pkg/shardservice/types.go b/pkg/shardservice/types.go index 933f68544f27..ed1ab119c66b 100644 --- a/pkg/shardservice/types.go +++ b/pkg/shardservice/types.go @@ -65,7 +65,7 @@ type ShardServer interface { // ShardService is sharding service. Each CN node holds an instance of the // ShardService. type ShardService interface { - // GetConfig returns the configuration of the shard service. + // Config returns the configuration of the shard service. Config() Config // GetStorage returns the storage of the shard service. GetStorage() ShardStorage @@ -166,9 +166,12 @@ const ( ReadApproxObjectsNum = 5 ReadPrimaryKeysMayBeModified = 6 ReadGetColumMetadataScanInfo = 7 - ReadReader = 8 + ReadBuildReader = 8 ReadMergeObjects = 9 ReadVisibleObjectStats = 10 + ReadNext = 11 + ReadClose = 12 + ReadCollectTombstones = 13 ) type ReadRequest struct { diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index ee664ec19531..78f33eac59a4 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -3643,8 +3643,9 @@ func collectTombstones( ctx := c.proc.GetTopContext() txnOp = c.proc.GetTxnOperator() if n.ScanSnapshot != nil && n.ScanSnapshot.TS != nil { - if !n.ScanSnapshot.TS.Equal(timestamp.Timestamp{LogicalTime: 0, PhysicalTime: 0}) && - n.ScanSnapshot.TS.Less(c.proc.GetTxnOperator().Txn().SnapshotTS) { + zeroTS := timestamp.Timestamp{LogicalTime: 0, PhysicalTime: 0} + snapTS := c.proc.GetTxnOperator().Txn().SnapshotTS + if !n.ScanSnapshot.TS.Equal(zeroTS) && n.ScanSnapshot.TS.Less(snapTS) { if c.proc.GetCloneTxnOperator() != nil { txnOp = c.proc.GetCloneTxnOperator() } else { @@ -3673,7 +3674,7 @@ func collectTombstones( if err != nil { return nil, err } - tombstone, err = rel.CollectTombstones(ctx, c.TxnOffset) + tombstone, err = rel.CollectTombstones(ctx, c.TxnOffset, engine.Policy_CollectAllTombstones) if err != nil { return nil, err } @@ -3686,7 +3687,7 @@ func collectTombstones( if err != nil { return nil, err } - subTombstone, err := subrelation.CollectTombstones(ctx, c.TxnOffset) + subTombstone, err := subrelation.CollectTombstones(ctx, c.TxnOffset, engine.Policy_CollectAllTombstones) if err != nil { return nil, err } @@ -3705,7 +3706,7 @@ func collectTombstones( if err != nil { return nil, err } - subTombstone, err := subrelation.CollectTombstones(ctx, c.TxnOffset) + subTombstone, err := subrelation.CollectTombstones(ctx, c.TxnOffset, engine.Policy_CollectAllTombstones) if err != nil { return nil, err } diff --git a/pkg/sql/compile/scope.go b/pkg/sql/compile/scope.go index ec899c4c0a2c..5fff2ddaa98f 100644 --- a/pkg/sql/compile/scope.go +++ b/pkg/sql/compile/scope.go @@ -895,7 +895,7 @@ func (s *Scope) buildReaders(c *Compile) (readers []engine.Reader, err error) { var subBlkList engine.RelData if s.NodeInfo.Data == nil || s.NodeInfo.Data.DataCnt() <= 1 { - //Even subBlkList is nil, + //Even if subBlkList is nil, //we still need to build reader for sub partition table to read data from memory. subBlkList = nil } else { diff --git a/pkg/vm/engine/disttae/datasource.go b/pkg/vm/engine/disttae/datasource.go index af515c450156..354d30b85648 100644 --- a/pkg/vm/engine/disttae/datasource.go +++ b/pkg/vm/engine/disttae/datasource.go @@ -64,11 +64,15 @@ func NewLocalDataSource( table *txnTable, txnOffset int, rangesSlice objectio.BlockInfoSlice, + extraTombstones engine.Tombstoner, skipReadMem bool, policy engine.TombstoneApplyPolicy, + category engine.DataSourceType, ) (source *LocalDataSource, err error) { source = &LocalDataSource{} + source.category = category + source.extraTombstones = extraTombstones source.fs = table.getTxn().engine.fs source.ctx = ctx source.mp = table.proc.Load().Mp() @@ -84,15 +88,17 @@ func NewLocalDataSource( source.rangeSlice = rangesSlice } - state, err := table.getPartitionState(ctx) - if err != nil { - return nil, err + if source.category != engine.ShardingLocalDataSource { + state, err := table.getPartitionState(ctx) + if err != nil { + return nil, err + } + source.pState = state } source.table = table - source.pState = state source.txnOffset = txnOffset - source.snapshotTS = types.TimestampToTS(table.getTxn().op.SnapshotTS()) + source.snapshotTS = types.TimestampToTS(table.db.op.SnapshotTS()) source.iteratePhase = engine.InMem if skipReadMem { @@ -261,8 +267,10 @@ func (rs *RemoteDataSource) SetFilterZM(_ objectio.ZoneMap) { // -------------------------------------------------------------------------------- type LocalDataSource struct { - rangeSlice objectio.BlockInfoSlice - pState *logtailreplay.PartitionState + category engine.DataSourceType + extraTombstones engine.Tombstoner + rangeSlice objectio.BlockInfoSlice + pState *logtailreplay.PartitionState memPKFilter *MemPKFilter pStateRows struct { @@ -515,12 +523,16 @@ func (ls *LocalDataSource) iterateInMemData( outBatch.SetRowCount(0) - if err = ls.filterInMemUnCommittedInserts(ctx, seqNums, mp, outBatch); err != nil { - return err + if ls.category != engine.ShardingRemoteDataSource { + if err = ls.filterInMemUnCommittedInserts(ctx, seqNums, mp, outBatch); err != nil { + return err + } } - if err = ls.filterInMemCommittedInserts(ctx, colTypes, seqNums, mp, outBatch); err != nil { - return err + if ls.category != engine.ShardingLocalDataSource { + if err = ls.filterInMemCommittedInserts(ctx, colTypes, seqNums, mp, outBatch); err != nil { + return err + } } return nil @@ -795,6 +807,19 @@ func (ls *LocalDataSource) ApplyTombstones( var err error + if ls.category == engine.ShardingRemoteDataSource { + if ls.extraTombstones != nil { + rowsOffset = ls.extraTombstones.ApplyInMemTombstones(bid, rowsOffset, nil) + rowsOffset, err = ls.extraTombstones.ApplyPersistedTombstones(ctx, ls.fs, ls.snapshotTS, bid, rowsOffset, nil) + if err != nil { + return nil, err + } + } + if len(rowsOffset) == 0 { + return nil, nil + } + } + if ls.tombstonePolicy&engine.Policy_SkipUncommitedInMemory == 0 && dynamicPolicy&engine.Policy_SkipUncommitedInMemory == 0 { rowsOffset = ls.applyWorkspaceEntryDeletes(bid, rowsOffset, nil) @@ -845,6 +870,16 @@ func (ls *LocalDataSource) GetTombstones( deletedRows = &nulls.Nulls{} deletedRows.InitWithSize(8192) + if ls.category == engine.ShardingRemoteDataSource { + if ls.extraTombstones != nil { + ls.extraTombstones.ApplyInMemTombstones(bid, nil, deletedRows) + _, err = ls.extraTombstones.ApplyPersistedTombstones(ctx, ls.fs, ls.snapshotTS, bid, nil, deletedRows) + if err != nil { + return nil, err + } + } + } + if ls.tombstonePolicy&engine.Policy_SkipUncommitedInMemory == 0 { ls.applyWorkspaceEntryDeletes(bid, nil, deletedRows) } diff --git a/pkg/vm/engine/disttae/engine.go b/pkg/vm/engine/disttae/engine.go index 364da67a5d5e..08368fc2ecfb 100644 --- a/pkg/vm/engine/disttae/engine.go +++ b/pkg/vm/engine/disttae/engine.go @@ -41,13 +41,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/lockservice" "github.com/matrixorigin/matrixone/pkg/logservice" "github.com/matrixorigin/matrixone/pkg/logutil" - "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/metadata" "github.com/matrixorigin/matrixone/pkg/pb/plan" pb "github.com/matrixorigin/matrixone/pkg/pb/statsinfo" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" client2 "github.com/matrixorigin/matrixone/pkg/queryservice/client" - "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/txn/client" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/util/stack" @@ -554,43 +552,9 @@ func (e *Engine) New(ctx context.Context, op client.TxnOperator) error { e.us, nil, ) - - id := objectio.NewSegmentid() - bytes := types.EncodeUuid(id) - txn := &Transaction{ - op: op, - proc: proc, - engine: e, - //meta: op.TxnRef(), - idGen: e.idGen, - tnStores: e.getTNServices(), - tableCache: new(sync.Map), - databaseMap: new(sync.Map), - deletedDatabaseMap: new(sync.Map), - tableOps: newTableOps(), - tablesInVain: make(map[uint64]int), - rowId: [6]uint32{ - types.DecodeUint32(bytes[0:4]), - types.DecodeUint32(bytes[4:8]), - types.DecodeUint32(bytes[8:12]), - types.DecodeUint32(bytes[12:16]), - 0, - 0, - }, - segId: *id, - deletedBlocks: &deletedBlocks{ - offsets: map[types.Blockid][]int64{}, - }, - cnBlkId_Pos: map[types.Blockid]Pos{}, - batchSelectList: make(map[*batch.Batch][]int64), - toFreeBatches: make(map[tableKey][]*batch.Batch), - syncCommittedTSCount: e.cli.GetSyncLatestCommitTSTimes(), - } - - txn.readOnly.Store(true) - // transaction's local segment for raw batch. - colexec.Get().PutCnSegment(id, colexec.TxnWorkSpaceIdType) + txn := NewTxnWorkSpace(e, proc) op.AddWorkspace(txn) + txn.BindTxnOp(op) e.pClient.validLogTailMustApplied(txn.op.SnapshotTS()) return nil diff --git a/pkg/vm/engine/disttae/logtail_consumer.go b/pkg/vm/engine/disttae/logtail_consumer.go index aeef5513e8ed..73fde428f3de 100644 --- a/pkg/vm/engine/disttae/logtail_consumer.go +++ b/pkg/vm/engine/disttae/logtail_consumer.go @@ -23,6 +23,7 @@ import ( "time" "github.com/fagongzi/goetty/v2" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/morpc" diff --git a/pkg/vm/engine/disttae/merge.go b/pkg/vm/engine/disttae/merge.go index ef5a1fcd2b90..60ffcda42096 100644 --- a/pkg/vm/engine/disttae/merge.go +++ b/pkg/vm/engine/disttae/merge.go @@ -85,7 +85,12 @@ func newCNMergeTask( targetObjSize uint32, ) (*cnMergeTask, error) { relData := NewBlockListRelationData(1) - source, err := tbl.buildLocalDataSource(ctx, 0, relData, engine.Policy_CheckAll) + source, err := tbl.buildLocalDataSource( + ctx, + 0, + relData, + engine.Policy_CheckAll, + engine.GeneralLocalDataSource) if err != nil { return nil, err } diff --git a/pkg/vm/engine/disttae/txn_table.go b/pkg/vm/engine/disttae/txn_table.go index 1d34df73caaa..47f500bbb758 100644 --- a/pkg/vm/engine/disttae/txn_table.go +++ b/pkg/vm/engine/disttae/txn_table.go @@ -456,77 +456,93 @@ func (tbl *txnTable) GetEngineType() engine.EngineType { return engine.Disttae } +func (tbl *txnTable) GetProcess() any { + return tbl.proc.Load() +} + func (tbl *txnTable) resetSnapshot() { tbl._partState.Store(nil) } func (tbl *txnTable) CollectTombstones( - ctx context.Context, txnOffset int, + ctx context.Context, + txnOffset int, + policy engine.TombstoneCollectPolicy, ) (engine.Tombstoner, error) { tombstone := NewEmptyTombstoneData() - offset := txnOffset - if tbl.db.op.IsSnapOp() { - offset = tbl.getTxn().GetSnapshotWriteOffset() - } + //collect uncommitted tombstones - //collect in memory + if policy&engine.Policy_CollectUncommittedTombstones != 0 { - //collect uncommitted in-memory tombstones from txn.writes - tbl.getTxn().ForEachTableWrites(tbl.db.databaseId, tbl.tableId, - offset, func(entry Entry) { - if entry.typ == INSERT { - return - } - //entry.typ == DELETE - if entry.bat.GetVector(0).GetType().Oid == types.T_Rowid { - /* - CASE: - create table t1(a int); - begin; - truncate t1; //txnDatabase.Truncate will DELETE mo_tables - show tables; // t1 must be shown - */ - //if entry.IsGeneratedByTruncate() { - // return - //} - //deletes in txn.Write maybe comes from PartitionState.Rows , - // PartitionReader need to skip them. - vs := vector.MustFixedColWithTypeCheck[types.Rowid](entry.bat.GetVector(0)) - tombstone.rowids = append(tombstone.rowids, vs...) - } - }) + offset := txnOffset + if tbl.db.op.IsSnapOp() { + offset = tbl.getTxn().GetSnapshotWriteOffset() + } - //collect uncommitted in-memory tombstones belongs to blocks persisted by CN writing S3 - tbl.getTxn().deletedBlocks.getDeletedRowIDs(&tombstone.rowids) + tbl.getTxn().ForEachTableWrites(tbl.db.databaseId, tbl.tableId, + offset, func(entry Entry) { + if entry.typ == INSERT { + return + } + //entry.typ == DELETE + if entry.bat.GetVector(0).GetType().Oid == types.T_Rowid { + /* + CASE: + create table t1(a int); + begin; + truncate t1; //txnDatabase.Truncate will DELETE mo_tables + show tables; // t1 must be shown + */ + //if entry.IsGeneratedByTruncate() { + // return + //} + //deletes in txn.Write maybe comes from PartitionState.Rows , + // PartitionReader need to skip them. + vs := vector.MustFixedColWithTypeCheck[types.Rowid](entry.bat.GetVector(0)) + tombstone.rowids = append(tombstone.rowids, vs...) + } + }) - //collect committed in-memory tombstones from partition state. - state, err := tbl.getPartitionState(ctx) - if err != nil { - return nil, err - } - { - ts := tbl.db.op.SnapshotTS() - iter := state.NewRowsIter(types.TimestampToTS(ts), nil, true) - for iter.Next() { - entry := iter.Entry() - //bid, o := entry.RowID.Decode() - tombstone.rowids = append(tombstone.rowids, entry.RowID) + //collect uncommitted in-memory tombstones belongs to blocks persisted by CN writing S3 + tbl.getTxn().deletedBlocks.getDeletedRowIDs(&tombstone.rowids) + + //collect uncommitted persisted tombstones. + if err := tbl.getTxn().getUncommittedS3Tombstone(&tombstone.files); err != nil { + return nil, err } - iter.Close() } - //collect uncommitted persisted tombstones. - if err := tbl.getTxn().getUncommittedS3Tombstone(&tombstone.files); err != nil { - return nil, err - } + //collect committed tombstones. + + if policy&engine.Policy_CollectCommittedTombstones != 0 { + //collect committed in-memory tombstones from partition state. + state, err := tbl.getPartitionState(ctx) + if err != nil { + return nil, err + } + { + ts := tbl.db.op.SnapshotTS() + iter := state.NewRowsIter(types.TimestampToTS(ts), nil, true) + for iter.Next() { + entry := iter.Entry() + //bid, o := entry.RowID.Decode() + tombstone.rowids = append(tombstone.rowids, entry.RowID) + } + iter.Close() + } + + //tombstone.SortInMemory() + //collect committed persisted tombstones from partition state. + snapshot := types.TimestampToTS(tbl.db.op.Txn().SnapshotTS) + err = state.CollectTombstoneObjects(snapshot, &tombstone.files) + if err != nil { + return nil, err + } + } tombstone.SortInMemory() - //collect committed persisted tombstones from partition state. - //state.GetTombstoneDeltaLocs(tombstone.blk2CommitLoc) - snapshot := types.TimestampToTS(tbl.db.op.Txn().SnapshotTS) - err = state.CollectTombstoneObjects(snapshot, &tombstone.files) - return tombstone, err + return tombstone, nil } // Ranges returns all unmodified blocks from the table. @@ -539,10 +555,11 @@ func (tbl *txnTable) Ranges( exprs []*plan.Expr, txnOffset int, ) (data engine.RelData, err error) { + unCommittedObjs, _ := tbl.collectUnCommittedDataObjs(txnOffset) return tbl.doRanges( ctx, exprs, - tbl.collectUnCommittedObjects(txnOffset), + unCommittedObjs, ) } @@ -845,8 +862,9 @@ func (tbl *txnTable) rangesOnePart( // Parameters: // - txnOffset: Transaction writes offset used to specify the starting position for reading data. // - fromSnapshot: Boolean indicating if the data is from a snapshot. -func (tbl *txnTable) collectUnCommittedObjects(txnOffset int) []objectio.ObjectStats { +func (tbl *txnTable) collectUnCommittedDataObjs(txnOffset int) ([]objectio.ObjectStats, map[objectio.ObjectNameShort]struct{}) { var unCommittedObjects []objectio.ObjectStats + unCommittedObjNames := make(map[objectio.ObjectNameShort]struct{}) if tbl.db.op.IsSnapOp() { txnOffset = tbl.getTxn().GetSnapshotWriteOffset() @@ -868,10 +886,11 @@ func (tbl *txnTable) collectUnCommittedObjects(txnOffset int) []objectio.ObjectS for i := 0; i < entry.bat.Vecs[1].Length(); i++ { stats.UnMarshal(entry.bat.Vecs[1].GetBytesAt(i)) unCommittedObjects = append(unCommittedObjects, stats) + unCommittedObjNames[*stats.ObjectShortName()] = struct{}{} } }) - return unCommittedObjects + return unCommittedObjects, unCommittedObjNames } //func (tbl *txnTable) collectDirtyBlocks( @@ -1584,7 +1603,7 @@ func buildRemoteDS( relData engine.RelData, ) (source engine.DataSource, err error) { - tombstones, err := tbl.CollectTombstones(ctx, txnOffset) + tombstones, err := tbl.CollectTombstones(ctx, txnOffset, engine.Policy_CollectAllTombstones) if err != nil { return nil, err } @@ -1630,7 +1649,12 @@ func BuildLocalDataSource( tbl = rel.(*txnTableDelegate).origin } - return tbl.buildLocalDataSource(ctx, txnOffset, ranges, engine.Policy_CheckAll) + return tbl.buildLocalDataSource( + ctx, + txnOffset, + ranges, + engine.Policy_CheckAll, + engine.GeneralLocalDataSource) } func (tbl *txnTable) buildLocalDataSource( @@ -1638,6 +1662,7 @@ func (tbl *txnTable) buildLocalDataSource( txnOffset int, relData engine.RelData, policy engine.TombstoneApplyPolicy, + category engine.DataSourceType, ) (source engine.DataSource, err error) { switch relData.GetType() { @@ -1666,8 +1691,10 @@ func (tbl *txnTable) buildLocalDataSource( tbl, txnOffset, ranges, + relData.GetTombstones(), skipReadMem, policy, + category, ) } @@ -1734,7 +1761,7 @@ func (tbl *txnTable) BuildReaders( shard = relData.DataSlice(current, current+divide) current = current + divide } - ds, err := tbl.buildLocalDataSource(ctx, txnOffset, shard, tombstonePolicy) + ds, err := tbl.buildLocalDataSource(ctx, txnOffset, shard, tombstonePolicy, engine.GeneralLocalDataSource) if err != nil { return nil, err } @@ -1757,6 +1784,19 @@ func (tbl *txnTable) BuildReaders( return rds, nil } +func (tbl *txnTable) BuildShardingReaders( + ctx context.Context, + p any, + expr *plan.Expr, + relData engine.RelData, + num int, + txnOffset int, + orderBy bool, + tombstonePolicy engine.TombstoneApplyPolicy, +) ([]engine.Reader, error) { + panic("Not Support") +} + func (tbl *txnTable) getPartitionState( ctx context.Context, ) (*logtailreplay.PartitionState, error) { diff --git a/pkg/vm/engine/disttae/txn_table_sharding.go b/pkg/vm/engine/disttae/txn_table_sharding.go index 2de610ceb07b..b2eadd732572 100644 --- a/pkg/vm/engine/disttae/txn_table_sharding.go +++ b/pkg/vm/engine/disttae/txn_table_sharding.go @@ -18,7 +18,9 @@ import ( "context" "github.com/fagongzi/goetty/v2/buf" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" @@ -31,11 +33,20 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/shard" pb "github.com/matrixorigin/matrixone/pkg/pb/statsinfo" "github.com/matrixorigin/matrixone/pkg/shardservice" + plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/cache" "github.com/matrixorigin/matrixone/pkg/vm/process" ) +type ReaderPhase uint8 + +const ( + InLocal ReaderPhase = iota + InRemote + InEnd +) + func newTxnTableWithItem( db *txnDatabase, item cache.TableItem, @@ -76,6 +87,35 @@ type txnTableDelegate struct { tableID uint64 is bool } + isMock bool +} + +func MockTableDelegate( + tableDelegate engine.Relation, + service shardservice.ShardService, +) (engine.Relation, error) { + delegate := tableDelegate.(*txnTableDelegate) + tbl := &txnTableDelegate{ + origin: delegate.origin, + isMock: true, + } + tbl.shard.service = service + + tbl.shard.service = service + tbl.shard.is = false + + if service.Config().Enable && + tbl.origin.db.databaseId != catalog.MO_CATALOG_ID { + tableID, policy, is, err := service.GetShardInfo(tbl.origin.tableId) + if err != nil { + return nil, err + } + + tbl.shard.is = is + tbl.shard.policy = policy + tbl.shard.tableID = tableID + } + return tbl, nil } func newTxnTable( @@ -211,7 +251,7 @@ func (tbl *txnTableDelegate) Ranges( buf := morpc.NewBuffer() defer buf.Close() - uncommitted := tbl.origin.collectUnCommittedObjects(txnOffset) + uncommitted, _ := tbl.origin.collectUnCommittedDataObjs(txnOffset) buf.Mark() for _, v := range uncommitted { buf.EncodeBytes(v[:]) @@ -247,15 +287,44 @@ func (tbl *txnTableDelegate) Ranges( func (tbl *txnTableDelegate) CollectTombstones( ctx context.Context, - txnOffset int) (engine.Tombstoner, error) { + txnOffset int, + policy engine.TombstoneCollectPolicy) (engine.Tombstoner, error) { if tbl.isLocal() { return tbl.origin.CollectTombstones( ctx, txnOffset, + policy, ) } - // TODO: forward - return nil, nil + + localTombstones, err := tbl.origin.CollectTombstones( + ctx, + txnOffset, + engine.Policy_CollectUncommittedTombstones, + ) + if err != nil { + return nil, err + } + var remoteTombstones engine.Tombstoner + err = tbl.forwardRead( + ctx, + shardservice.ReadCollectTombstones, + func(param *shard.ReadParam) { + param.CollectTombstonesParam.CollectPolicy = engine.Policy_CollectCommittedTombstones + }, + func(resp []byte) { + tombstones, err := UnmarshalTombstoneData(resp) + if err != nil { + panic(err) + } + remoteTombstones = tombstones + }, + ) + if err != nil { + return nil, err + } + localTombstones.Merge(remoteTombstones) + return localTombstones, nil } func (tbl *txnTableDelegate) GetColumMetadataScanInfo( @@ -338,7 +407,278 @@ func (tbl *txnTableDelegate) BuildReaders( engine.Policy_CheckAll, ) } - return nil, nil + return tbl.BuildShardingReaders( + ctx, + proc, + expr, + relData, + num, + txnOffset, + orderBy, + engine.Policy_CheckAll, + ) +} + +type shardingLocalReader struct { + iteratePhase ReaderPhase + closed bool + lrd engine.Reader + tblDelegate *txnTableDelegate + streamID types.Uuid + //relation data to distribute to remote CN which holds shard's partition state. + remoteRelData engine.RelData + remoteTombApplyPolicy engine.TombstoneApplyPolicy + remoteScanType int +} + +// TODO:: +func MockShardingLocalReader() engine.Reader { + return &shardingLocalReader{} +} + +func (r *shardingLocalReader) Read( + ctx context.Context, + cols []string, + expr *plan.Expr, + mp *mpool.MPool, + bat *batch.Batch, +) (isEnd bool, err error) { + defer func() { + if err != nil || isEnd { + r.close() + } + }() + + for { + + switch r.iteratePhase { + case InLocal: + if r.lrd != nil { + isEnd, err = r.lrd.Read(ctx, cols, expr, mp, bat) + if err != nil { + return + } + if !isEnd { + return + } + } + if r.remoteRelData == nil || r.remoteRelData.DataCnt() == 0 { + r.iteratePhase = InEnd + return + } + relData, err := r.remoteRelData.MarshalBinary() + if err != nil { + return false, err + } + err = r.tblDelegate.forwardRead( + ctx, + shardservice.ReadBuildReader, + func(param *shard.ReadParam) { + param.ReaderBuildParam.RelData = relData + param.ReaderBuildParam.Expr = expr + param.ReaderBuildParam.ScanType = int32(r.remoteScanType) + param.ReaderBuildParam.TombstoneApplyPolicy = int32(r.remoteTombApplyPolicy) + }, + func(resp []byte) { + r.streamID = types.DecodeUuid(resp) + }, + ) + if err != nil { + return false, err + } + r.iteratePhase = InRemote + case InRemote: + err = r.tblDelegate.forwardRead( + ctx, + shardservice.ReadNext, + func(param *shard.ReadParam) { + param.ReadNextParam.Uuid = types.EncodeUuid(&r.streamID) + param.ReadNextParam.Columns = cols + }, + func(resp []byte) { + isEnd = types.DecodeBool(resp) + if isEnd { + return + } + resp = resp[1:] + l := types.DecodeUint32(resp) + resp = resp[4:] + if err := bat.UnmarshalBinary(resp[:l]); err != nil { + panic(err) + } + }, + ) + if err != nil { + return false, err + } + if isEnd { + r.iteratePhase = InEnd + } + return + case InEnd: + return true, nil + } + + } + +} + +func (r *shardingLocalReader) Close() error { + return r.close() +} + +func (r *shardingLocalReader) close() error { + if !r.closed { + if r.lrd != nil { + r.lrd.Close() + } + if r.remoteRelData != nil { + err := r.tblDelegate.forwardRead( + context.Background(), + shardservice.ReadClose, + func(param *shard.ReadParam) { + param.ReadCloseParam.Uuid = types.EncodeUuid(&r.streamID) + }, + func(resp []byte) { + }, + ) + if err != nil { + return err + } + } + } + return nil +} + +func (r *shardingLocalReader) SetOrderBy(orderby []*plan.OrderBySpec) { + panic("not implemented") +} + +func (r *shardingLocalReader) GetOrderBy() []*plan.OrderBySpec { + panic("not implemented") +} + +func (r *shardingLocalReader) SetFilterZM(zm objectio.ZoneMap) { + panic("not implemented") +} + +func (tbl *txnTableDelegate) BuildShardingReaders( + ctx context.Context, + p any, + expr *plan.Expr, + relData engine.RelData, + num int, + txnOffset int, + orderBy bool, + policy engine.TombstoneApplyPolicy, +) ([]engine.Reader, error) { + var rds []engine.Reader + proc := p.(*process.Process) + + if plan2.IsFalseExpr(expr) { + return []engine.Reader{new(emptyReader)}, nil + } + + if orderBy && num != 1 { + return nil, moerr.NewInternalErrorNoCtx("orderBy only support one reader") + } + + _, uncommittedObjNames := tbl.origin.collectUnCommittedDataObjs(txnOffset) + uncommittedTombstones, err := tbl.origin.CollectTombstones( + ctx, + txnOffset, + engine.Policy_CollectUncommittedTombstones) + if err != nil { + return nil, err + } + group := func(rd engine.RelData) (local engine.RelData, remote engine.RelData) { + local = rd.BuildEmptyRelData() + remote = rd.BuildEmptyRelData() + engine.ForRangeBlockInfo(0, rd.DataCnt(), rd, func(bi objectio.BlockInfo) (bool, error) { + if bi.IsMemBlk() { + local.AppendBlockInfo(&bi) + remote.AppendBlockInfo(&bi) + return true, nil + } + if _, ok := uncommittedObjNames[*objectio.ShortName(&bi.BlockID)]; ok { + local.AppendBlockInfo(&bi) + } else { + remote.AppendBlockInfo(&bi) + } + return true, nil + }) + return + } + + //relData maybe is nil, indicate that only read data from memory. + if relData == nil || relData.DataCnt() == 0 { + relData = NewBlockListRelationData(1) + } + + blkCnt := relData.DataCnt() + newNum := num + if blkCnt < num { + newNum = blkCnt + for i := 0; i < num-blkCnt; i++ { + rds = append(rds, new(emptyReader)) + } + } + + scanType := determineScanType(relData, newNum) + mod := blkCnt % newNum + divide := blkCnt / newNum + var shard engine.RelData + for i := 0; i < newNum; i++ { + if i == 0 { + shard = relData.DataSlice(i*divide, (i+1)*divide+mod) + } else { + shard = relData.DataSlice(i*divide+mod, (i+1)*divide+mod) + } + + localRelData, remoteRelData := group(shard) + + srd := &shardingLocalReader{ + //lrd: lrd, + tblDelegate: tbl, + //remoteRelData: remoteRelData, + remoteTombApplyPolicy: engine.Policy_SkipUncommitedInMemory | engine.Policy_SkipUncommitedS3, + remoteScanType: scanType, + } + + if localRelData.DataCnt() > 0 { + ds, err := tbl.origin.buildLocalDataSource( + ctx, + txnOffset, + localRelData, + policy|engine.Policy_SkipCommittedInMemory|engine.Policy_SkipCommittedS3, + engine.ShardingLocalDataSource) + if err != nil { + return nil, err + } + lrd, err := NewReader( + ctx, + proc, + tbl.origin.getTxn().engine, + tbl.origin.GetTableDef(ctx), + tbl.origin.db.op.SnapshotTS(), + expr, + ds, + ) + if err != nil { + return nil, err + } + lrd.scanType = scanType + srd.lrd = lrd + } + + if remoteRelData.DataCnt() > 0 { + remoteRelData.AttachTombstones(uncommittedTombstones) + srd.remoteRelData = remoteRelData + } + rds = append(rds, srd) + } + + return rds, nil } func (tbl *txnTableDelegate) PrimaryKeysMayBeModified( @@ -572,6 +912,10 @@ func (tbl *txnTableDelegate) GetEngineType() engine.EngineType { return tbl.origin.GetEngineType() } +func (tbl *txnTableDelegate) GetProcess() any { + return tbl.origin.GetProcess() +} + func (tbl *txnTableDelegate) isLocal() bool { return !tbl.shard.service.Config().Enable || // sharding not enabled !tbl.shard.is || // normal table @@ -611,6 +955,43 @@ func (tbl *txnTableDelegate) getReadRequest( }, nil } +// Just for UT. +func (tbl *txnTableDelegate) mockForwardRead( + ctx context.Context, + method int, + request shardservice.ReadRequest, +) ([]byte, error) { + + handles := map[int]shardservice.ReadFunc{ + shardservice.ReadRows: HandleShardingReadRows, + shardservice.ReadSize: HandleShardingReadSize, + shardservice.ReadStats: HandleShardingReadStatus, + shardservice.ReadApproxObjectsNum: HandleShardingReadApproxObjectsNum, + shardservice.ReadRanges: HandleShardingReadRanges, + shardservice.ReadGetColumMetadataScanInfo: HandleShardingReadGetColumMetadataScanInfo, + shardservice.ReadBuildReader: HandleShardingReadBuildReader, + shardservice.ReadPrimaryKeysMayBeModified: HandleShardingReadPrimaryKeysMayBeModified, + shardservice.ReadMergeObjects: HandleShardingReadMergeObjects, + shardservice.ReadVisibleObjectStats: HandleShardingReadVisibleObjectStats, + shardservice.ReadClose: HandleShardingReadClose, + shardservice.ReadNext: HandleShardingReadNext, + shardservice.ReadCollectTombstones: HandleShardingReadCollectTombstones, + } + buf := morpc.NewBuffer() + resp, err := handles[method]( + ctx, + shard.TableShard{}, + tbl.origin.getEngine(), + request.Param, + tbl.origin.db.op.SnapshotTS(), + buf, + ) + if err != nil { + return nil, err + } + return resp, nil +} + func (tbl *txnTableDelegate) forwardRead( ctx context.Context, method int, @@ -627,6 +1008,15 @@ func (tbl *txnTableDelegate) forwardRead( applyParam(&request.Param) + if tbl.isMock { + res, err := tbl.mockForwardRead(ctx, method, request) + if err != nil { + return err + } + apply(res) + return nil + } + shardID := uint64(0) switch tbl.shard.policy { case shard.Policy_Partition: diff --git a/pkg/vm/engine/disttae/txn_table_sharding_handle.go b/pkg/vm/engine/disttae/txn_table_sharding_handle.go index a6b90deeefbb..9d46ef348248 100644 --- a/pkg/vm/engine/disttae/txn_table_sharding_handle.go +++ b/pkg/vm/engine/disttae/txn_table_sharding_handle.go @@ -17,8 +17,14 @@ package disttae import ( "bytes" "context" + "strings" + "sync" + "time" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/morpc" + "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/objectio" @@ -26,9 +32,66 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/shard" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/gc" "github.com/matrixorigin/matrixone/pkg/vm/process" ) +const StreamReaderLease = time.Minute * 2 + +type shardingRemoteReader struct { + streamID types.Uuid + rd engine.Reader + colTypes []types.Type + deadline time.Time +} + +func (sr *shardingRemoteReader) updateCols(cols []string, tblDef *plan.TableDef) { + if len(sr.colTypes) == 0 { + sr.colTypes = make([]types.Type, len(cols)) + for i, column := range cols { + column = strings.ToLower(column) + if column == catalog.Row_ID { + sr.colTypes[i] = objectio.RowidType + } else { + colIdx := tblDef.Name2ColIndex[column] + colDef := tblDef.Cols[colIdx] + sr.colTypes[i] = types.T(colDef.Typ.Id).ToType() + sr.colTypes[i].Scale = colDef.Typ.Scale + sr.colTypes[i].Width = colDef.Typ.Width + } + } + } +} + +type streamHandle struct { + sync.Mutex + streamReaders map[types.Uuid]shardingRemoteReader + GCManager *gc.Manager +} + +var streamHandler streamHandle + +func init() { + streamHandler.streamReaders = make(map[types.Uuid]shardingRemoteReader) + streamHandler.GCManager = gc.NewManager( + gc.WithCronJob( + "streamReaderGC", + StreamReaderLease, + func(ctx context.Context) error { + streamHandler.Lock() + defer streamHandler.Unlock() + for id, sr := range streamHandler.streamReaders { + if time.Now().After(sr.deadline) { + delete(streamHandler.streamReaders, id) + } + } + return nil + }, + ), + ) + +} + // HandleShardingReadRows handles sharding read rows func HandleShardingReadRows( ctx context.Context, @@ -182,8 +245,69 @@ func HandleShardingReadRanges( return buffer.EncodeBytes(bys), nil } -// HandleShardingReadGetColumMetadataScanInfo handles sharding read GetColumMetadataScanInfo -func HandleShardingReadGetColumMetadataScanInfo( +// HandleShardingReadReader handles sharding read Reader +func HandleShardingReadBuildReader( + ctx context.Context, + shard shard.TableShard, + e engine.Engine, + param shard.ReadParam, + ts timestamp.Timestamp, + buffer *morpc.Buffer, +) ([]byte, error) { + tbl, err := getTxnTable( + ctx, + param, + e, + ) + if err != nil { + return nil, err + } + + relData, err := UnmarshalRelationData(param.ReaderBuildParam.RelData) + if err != nil { + return nil, err + } + + ds, err := tbl.buildLocalDataSource( + ctx, + 0, + relData, + engine.TombstoneApplyPolicy(param.ReaderBuildParam.TombstoneApplyPolicy), + engine.ShardingRemoteDataSource, + ) + if err != nil { + return nil, err + } + + rd, err := NewReader( + ctx, + tbl.proc.Load(), + e.(*Engine), + tbl.tableDef, + tbl.db.op.SnapshotTS(), + param.ReaderBuildParam.Expr, + ds, + ) + if err != nil { + return nil, err + } + + uuid, err := types.BuildUuid() + if err != nil { + return nil, err + } + streamHandler.Lock() + defer streamHandler.Unlock() + streamHandler.streamReaders[uuid] = shardingRemoteReader{ + streamID: uuid, + rd: rd, + deadline: time.Now().Add(StreamReaderLease), + } + + return buffer.EncodeBytes(types.EncodeUuid(&uuid)), nil +} + +func HandleShardingReadNext( ctx context.Context, shard shard.TableShard, engine engine.Engine, @@ -191,6 +315,7 @@ func HandleShardingReadGetColumMetadataScanInfo( ts timestamp.Timestamp, buffer *morpc.Buffer, ) ([]byte, error) { + tbl, err := getTxnTable( ctx, param, @@ -199,30 +324,94 @@ func HandleShardingReadGetColumMetadataScanInfo( if err != nil { return nil, err } + mp := tbl.proc.Load().Mp() - infos, err := tbl.GetColumMetadataScanInfo( + streamID := types.DecodeUuid(param.ReadNextParam.Uuid) + cols := param.ReadNextParam.Columns + //find reader by streamID + streamHandler.Lock() + sr, ok := streamHandler.streamReaders[streamID] + if !ok { + streamHandler.Unlock() + return nil, moerr.NewInternalErrorNoCtx("stream reader not found, may be expired") + } + streamHandler.Unlock() + sr.deadline = time.Now().Add(StreamReaderLease) + + sr.updateCols(cols, tbl.tableDef) + + buildBatch := func() *batch.Batch { + bat := batch.NewWithSize(len(sr.colTypes)) + bat.Attrs = append(bat.Attrs, cols...) + + for i := 0; i < len(sr.colTypes); i++ { + bat.Vecs[i] = vector.NewVec(sr.colTypes[i]) + + } + return bat + } + bat := buildBatch() + defer func() { + bat.Clean(mp) + }() + + isEnd, err := sr.rd.Read( ctx, - param.GetColumMetadataScanInfoParam.ColumnName, + cols, + nil, + mp, + bat, ) if err != nil { return nil, err } + if isEnd { + return buffer.EncodeBytes(types.EncodeBool(&isEnd)), nil + } - v := plan.MetadataScanInfos{ - Infos: infos, + var w bytes.Buffer + if _, err := w.Write(types.EncodeBool(&isEnd)); err != nil { + return nil, err } - bys, err := v.Marshal() + encBat, err := bat.MarshalBinary() if err != nil { - panic(err) + return nil, err } - return buffer.EncodeBytes(bys), nil + l := uint32(len(encBat)) + if _, err := w.Write(types.EncodeUint32(&l)); err != nil { + return nil, err + } + if _, err := w.Write(encBat); err != nil { + return nil, err + } + return buffer.EncodeBytes(w.Bytes()), nil } -// HandleShardingReadReader handles sharding read Reader -func HandleShardingReadReader( +func HandleShardingReadClose( ctx context.Context, shard shard.TableShard, - e engine.Engine, + engine engine.Engine, + param shard.ReadParam, + ts timestamp.Timestamp, + buffer *morpc.Buffer, +) ([]byte, error) { + streamID := types.DecodeUuid(param.ReadCloseParam.Uuid) + //find reader by streamID + streamHandler.Lock() + defer streamHandler.Unlock() + sr, ok := streamHandler.streamReaders[streamID] + if !ok { + return nil, moerr.NewInternalErrorNoCtx("stream reader not found, may be expired") + } + sr.rd.Close() + delete(streamHandler.streamReaders, sr.streamID) + return nil, nil +} + +func HandleShardingReadCollectTombstones( + ctx context.Context, + shard shard.TableShard, + eng engine.Engine, param shard.ReadParam, ts timestamp.Timestamp, buffer *morpc.Buffer, @@ -230,31 +419,63 @@ func HandleShardingReadReader( tbl, err := getTxnTable( ctx, param, - e, + eng, ) if err != nil { return nil, err } - relData, err := UnmarshalRelationData(param.ReaderParam.Ranges) + tombstones, err := tbl.CollectTombstones( + ctx, + 0, + engine.TombstoneCollectPolicy(param.CollectTombstonesParam.CollectPolicy), + ) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + err = tombstones.MarshalBinaryWithBuffer(&buf) if err != nil { return nil, err } - _, err = tbl.BuildReaders( + return buffer.EncodeBytes(buf.Bytes()), nil +} + +// HandleShardingReadGetColumMetadataScanInfo handles sharding read GetColumMetadataScanInfo +func HandleShardingReadGetColumMetadataScanInfo( + ctx context.Context, + shard shard.TableShard, + engine engine.Engine, + param shard.ReadParam, + ts timestamp.Timestamp, + buffer *morpc.Buffer, +) ([]byte, error) { + tbl, err := getTxnTable( ctx, - tbl.proc.Load(), - ¶m.ReaderParam.Expr, - relData, - int(param.ReaderParam.Num), - int(param.ReaderParam.TxnOffset), - param.ReaderParam.OrderedScan, - engine.Policy_CheckAll, + param, + engine, ) if err != nil { return nil, err } - // TODO: - return nil, nil + + infos, err := tbl.GetColumMetadataScanInfo( + ctx, + param.GetColumMetadataScanInfoParam.ColumnName, + ) + if err != nil { + return nil, err + } + + v := plan.MetadataScanInfos{ + Infos: infos, + } + bys, err := v.Marshal() + if err != nil { + panic(err) + } + return buffer.EncodeBytes(bys), nil } // HandleShardingReadPrimaryKeysMayBeModified handles sharding read PrimaryKeysMayBeModified @@ -392,6 +613,9 @@ func getTxnTable( if err != nil { return nil, err } + ws := NewTxnWorkSpace(engine.(*Engine), proc) + proc.GetTxnOperator().AddWorkspace(ws) + ws.BindTxnOp(proc.GetTxnOperator()) db := &txnDatabase{ op: proc.GetTxnOperator(), diff --git a/pkg/vm/engine/disttae/types.go b/pkg/vm/engine/disttae/types.go index 3172493b83fb..2eac352fd65a 100644 --- a/pkg/vm/engine/disttae/types.go +++ b/pkg/vm/engine/disttae/types.go @@ -24,6 +24,7 @@ import ( "sync/atomic" "time" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/vm/message" "github.com/panjf2000/ants/v2" @@ -225,6 +226,10 @@ type Engine struct { moColumnsCreatedTime *vector.Vector } +func (e *Engine) SetService(svr string) { + e.service = svr +} + func (txn *Transaction) String() string { return fmt.Sprintf("writes %v", txn.writes) } @@ -365,6 +370,43 @@ func (b *deletedBlocks) iter(fn func(*types.Blockid, []int64) bool) { } } +func NewTxnWorkSpace(eng *Engine, proc *process.Process) *Transaction { + id := objectio.NewSegmentid() + bytes := types.EncodeUuid(id) + txn := &Transaction{ + proc: proc, + engine: eng, + idGen: eng.idGen, + tnStores: eng.getTNServices(), + tableCache: new(sync.Map), + databaseMap: new(sync.Map), + deletedDatabaseMap: new(sync.Map), + tableOps: newTableOps(), + tablesInVain: make(map[uint64]int), + rowId: [6]uint32{ + types.DecodeUint32(bytes[0:4]), + types.DecodeUint32(bytes[4:8]), + types.DecodeUint32(bytes[8:12]), + types.DecodeUint32(bytes[12:16]), + 0, + 0, + }, + segId: *id, + deletedBlocks: &deletedBlocks{ + offsets: map[types.Blockid][]int64{}, + }, + cnBlkId_Pos: map[types.Blockid]Pos{}, + batchSelectList: make(map[*batch.Batch][]int64), + toFreeBatches: make(map[tableKey][]*batch.Batch), + syncCommittedTSCount: eng.cli.GetSyncLatestCommitTSTimes(), + } + + txn.readOnly.Store(true) + // transaction's local segment for raw batch. + colexec.Get().PutCnSegment(id, colexec.TxnWorkSpaceIdType) + return txn +} + func (txn *Transaction) PutCnBlockDeletes(blockId *types.Blockid, offsets []int64) { txn.deletedBlocks.addDeletedBlocks(blockId, offsets) } diff --git a/pkg/vm/engine/memoryengine/table_reader.go b/pkg/vm/engine/memoryengine/table_reader.go index fa665f4350cd..4fc1e9756f6e 100644 --- a/pkg/vm/engine/memoryengine/table_reader.go +++ b/pkg/vm/engine/memoryengine/table_reader.go @@ -42,6 +42,18 @@ type IterInfo struct { IterID ID } +func (t *Table) BuildShardingReaders( + ctx context.Context, + _ any, + expr *plan.Expr, + relData engine.RelData, + parallel int, + _ int, + _ bool, + _ engine.TombstoneApplyPolicy) (readers []engine.Reader, err error) { + panic("Not Support") +} + func (t *Table) BuildReaders( ctx context.Context, _ any, @@ -230,6 +242,10 @@ func (t *Table) GetEngineType() engine.EngineType { return engine.Memory } +func (t *Table) GetProcess() any { + panic("Not Support") +} + func (t *Table) Ranges(_ context.Context, _ []*plan.Expr, _ int) (engine.RelData, error) { rd := &MemRelationData{} nodes := getTNServices(t.engine.cluster) @@ -245,7 +261,11 @@ func (t *Table) Ranges(_ context.Context, _ []*plan.Expr, _ int) (engine.RelData return rd, nil } -func (t *Table) CollectTombstones(ctx context.Context, txnOffset int) (engine.Tombstoner, error) { +func (t *Table) CollectTombstones( + _ context.Context, + _ int, + _ engine.TombstoneCollectPolicy, +) (engine.Tombstoner, error) { panic("implement me") } diff --git a/pkg/vm/engine/memoryengine/table_test.go b/pkg/vm/engine/memoryengine/table_test.go new file mode 100644 index 000000000000..44debd16c04f --- /dev/null +++ b/pkg/vm/engine/memoryengine/table_test.go @@ -0,0 +1,42 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memoryengine + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Only for passing the UT coverage check. +func Test_TableReader(t *testing.T) { + table := new(Table) + assert.Panics(t, func() { + table.BuildShardingReaders( + context.TODO(), + nil, + nil, + nil, + 0, + 0, + false, + 0, + ) + }) + assert.Panics(t, func() { + table.GetProcess() + }) +} diff --git a/pkg/vm/engine/test/reader_test.go b/pkg/vm/engine/test/reader_test.go index 5c27eeeeb017..37f543600a86 100644 --- a/pkg/vm/engine/test/reader_test.go +++ b/pkg/vm/engine/test/reader_test.go @@ -21,9 +21,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -31,15 +33,20 @@ import ( "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/shard" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae" "github.com/matrixorigin/matrixone/pkg/vm/engine/engine_util" catalog2 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" + testutil2 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/testutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/iface/handle" "github.com/matrixorigin/matrixone/pkg/vm/engine/test/testutil" + "github.com/matrixorigin/matrixone/pkg/vm/process" ) func Test_ReaderCanReadRangesBlocksWithoutDeletes(t *testing.T) { @@ -382,3 +389,881 @@ func Test_ReaderCanReadCommittedInMemInsertAndDeletes(t *testing.T) { } } + +func Test_ShardingHandler(t *testing.T) { + var ( + //err error + //mp *mpool.MPool + accountId = catalog.System_Account + tableName = "test_reader_table" + databaseName = "test_reader_database" + + primaryKeyIdx int = 3 + + taeEngine *testutil.TestTxnStorage + rpcAgent *testutil.MockRPCAgent + disttaeEngine *testutil.TestDisttaeEngine + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ctx = context.WithValue(ctx, defines.TenantIDKey{}, accountId) + + // mock a schema with 4 columns and the 4th column as primary key + // the first column is the 9th column in the predefined columns in + // the mock function. Here we exepct the type of the primary key + // is types.T_char or types.T_varchar + schema := catalog2.MockSchemaEnhanced(4, primaryKeyIdx, 9) + schema.Name = tableName + + { + opt, err := testutil.GetS3SharedFileServiceOption(ctx, testutil.GetDefaultTestPath("test", t)) + require.NoError(t, err) + + disttaeEngine, taeEngine, rpcAgent, _ = testutil.CreateEngines( + ctx, + testutil.TestOptions{TaeEngineOptions: opt}, + t, + testutil.WithDisttaeEngineWorkspaceThreshold(mpool.MB*2), + testutil.WithDisttaeEngineInsertEntryMaxCount(10000), + ) + defer func() { + disttaeEngine.Close(ctx) + taeEngine.Close(true) + rpcAgent.Close() + }() + + _, _, err = disttaeEngine.CreateDatabaseAndTable(ctx, databaseName, tableName, schema) + require.NoError(t, err) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + } + + testutil2.CompactBlocks(t, 0, taeEngine.GetDB(), databaseName, schema, false) + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 1, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + var blkId *common.ID + for iter.Next() { + if !iter.GetObject().IsAppendable() { + continue + } + blkId = iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + } + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + //handle collect tombstones. + { + _, rel, txn, err := disttaeEngine.GetTable(ctx, databaseName, tableName) + require.NoError(t, err) + + pInfo, err := process.MockProcessInfoWithPro("", rel.GetProcess()) + require.NoError(t, err) + readerBuildParam := shard.ReadParam{ + Process: pInfo, + TxnTable: shard.TxnTable{ + DatabaseID: rel.GetDBID(ctx), + DatabaseName: databaseName, + AccountID: uint64(catalog.System_Account), + TableName: tableName, + }, + } + readerBuildParam.CollectTombstonesParam.CollectPolicy = + engine.Policy_CollectCommittedTombstones + + res, err := disttae.HandleShardingReadCollectTombstones( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readerBuildParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.NoError(t, err) + + tombstones, err := disttae.UnmarshalTombstoneData(res) + require.NoError(t, err) + + require.True(t, tombstones.HasAnyInMemoryTombstone()) + + readerBuildParam.GetColumMetadataScanInfoParam.ColumnName = + schema.ColDefs[primaryKeyIdx].Name + + var m plan.MetadataScanInfos + res, err = disttae.HandleShardingReadGetColumMetadataScanInfo( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readerBuildParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.NoError(t, err) + + err = m.Unmarshal(res) + require.NoError(t, err) + + require.NoError(t, txn.Commit(ctx)) + } + +} + +func Test_ShardingRemoteReader(t *testing.T) { + var ( + //err error + mp *mpool.MPool + accountId = catalog.System_Account + tableName = "test_reader_table" + databaseName = "test_reader_database" + + primaryKeyIdx int = 3 + + taeEngine *testutil.TestTxnStorage + rpcAgent *testutil.MockRPCAgent + disttaeEngine *testutil.TestDisttaeEngine + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ctx = context.WithValue(ctx, defines.TenantIDKey{}, accountId) + + // mock a schema with 4 columns and the 4th column as primary key + // the first column is the 9th column in the predefined columns in + // the mock function. Here we exepct the type of the primary key + // is types.T_char or types.T_varchar + schema := catalog2.MockSchemaEnhanced(4, primaryKeyIdx, 9) + schema.Name = tableName + + { + opt, err := testutil.GetS3SharedFileServiceOption(ctx, testutil.GetDefaultTestPath("test", t)) + require.NoError(t, err) + + disttaeEngine, taeEngine, rpcAgent, mp = testutil.CreateEngines( + ctx, + testutil.TestOptions{TaeEngineOptions: opt}, + t, + testutil.WithDisttaeEngineWorkspaceThreshold(mpool.MB*2), + testutil.WithDisttaeEngineInsertEntryMaxCount(10000), + ) + defer func() { + disttaeEngine.Close(ctx) + taeEngine.Close(true) + rpcAgent.Close() + }() + + _, _, err = disttaeEngine.CreateDatabaseAndTable(ctx, databaseName, tableName, schema) + require.NoError(t, err) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + } + + testutil2.CompactBlocks(t, 0, taeEngine.GetDB(), databaseName, schema, false) + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 1, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + var blkId *common.ID + for iter.Next() { + if !iter.GetObject().IsAppendable() { + continue + } + blkId = iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + } + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + + txn, _, reader, err := testutil.GetTableTxnReader( + ctx, + disttaeEngine, + databaseName, + tableName, + nil, + mp, + t, + ) + require.NoError(t, err) + + ret := batch.NewWithSize(1) + for _, col := range schema.ColDefs { + if col.Name == schema.ColDefs[primaryKeyIdx].Name { + vec := vector.NewVec(col.Type) + ret.Vecs[0] = vec + ret.Attrs = []string{col.Name} + break + } + } + _, err = reader.Read(ctx, []string{ + schema.ColDefs[primaryKeyIdx].Name}, nil, mp, ret) + require.NoError(t, err) + require.True(t, ret.Allocated() > 0) + + require.Equal(t, 2, ret.RowCount()) + require.NoError(t, txn.Commit(ctx)) + ret.Clean(mp) + } + + { + _, rel, txn, err := disttaeEngine.GetTable(ctx, databaseName, tableName) + require.NoError(t, err) + + pInfo, err := process.MockProcessInfoWithPro("", rel.GetProcess()) + require.NoError(t, err) + readerBuildParam := shard.ReadParam{ + Process: pInfo, + TxnTable: shard.TxnTable{ + DatabaseID: rel.GetDBID(ctx), + DatabaseName: databaseName, + AccountID: uint64(catalog.System_Account), + TableName: tableName, + }, + } + relData, err := rel.Ranges(ctx, nil, 0) + require.NoError(t, err) + //TODO:: attach tombstones. + //tombstones, err := rel.CollectTombstones( + // ctx, + // 0, + // engine.Policy_CollectAllTombstones) + data, err := relData.MarshalBinary() + require.NoError(t, err) + readerBuildParam.ReaderBuildParam.RelData = data + readerBuildParam.ReaderBuildParam.ScanType = disttae.SMALL + readerBuildParam.ReaderBuildParam.TombstoneApplyPolicy = + int32(engine.Policy_SkipUncommitedInMemory | engine.Policy_SkipUncommitedS3) + res, err := disttae.HandleShardingReadBuildReader( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readerBuildParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.NoError(t, err) + streamID := types.DecodeUuid(res) + //readNext + readNextParam := shard.ReadParam{ + Process: pInfo, + TxnTable: shard.TxnTable{ + DatabaseID: rel.GetDBID(ctx), + DatabaseName: databaseName, + AccountID: uint64(catalog.System_Account), + TableName: tableName, + }, + } + readNextParam.ReadNextParam.Uuid = types.EncodeUuid(&streamID) + readNextParam.ReadNextParam.Columns = []string{ + schema.ColDefs[primaryKeyIdx].Name} + buildBatch := func() *batch.Batch { + bat := batch.NewWithSize(1) + for _, col := range schema.ColDefs { + if col.Name == schema.ColDefs[primaryKeyIdx].Name { + vec := vector.NewVec(col.Type) + bat.Vecs[0] = vec + bat.Attrs = []string{col.Name} + break + } + } + return bat + } + rows := 0 + for { + bat := buildBatch() + res, err := disttae.HandleShardingReadNext( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readNextParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.NoError(t, err) + isEnd := types.DecodeBool(res) + if isEnd { + break + } + res = res[1:] + l := types.DecodeUint32(res) + res = res[4:] + if err := bat.UnmarshalBinary(res[:l]); err != nil { + panic(err) + } + rows += int(bat.RowCount()) + bat.Clean(mp) + } + require.Equal(t, 2, rows) + + readCloseParam := shard.ReadParam{ + Process: pInfo, + TxnTable: shard.TxnTable{ + DatabaseID: rel.GetDBID(ctx), + DatabaseName: databaseName, + AccountID: uint64(catalog.System_Account), + TableName: tableName, + }, + } + readCloseParam.ReadCloseParam.Uuid = types.EncodeUuid(&streamID) + _, err = disttae.HandleShardingReadClose( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readCloseParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.NoError(t, err) + + _, err = disttae.HandleShardingReadNext( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readNextParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.Error(t, err) + + _, err = disttae.HandleShardingReadClose( + ctx, + shard.TableShard{}, + disttaeEngine.Engine, + readCloseParam, + timestamp.Timestamp{}, + morpc.NewBuffer(), + ) + require.Error(t, err) + + require.NoError(t, txn.Commit(ctx)) + } + +} + +func Test_ShardingTableDelegate(t *testing.T) { + var ( + //err error + //mp *mpool.MPool + accountId = catalog.System_Account + tableName = "test_reader_table" + databaseName = "test_reader_database" + + primaryKeyIdx int = 3 + + taeEngine *testutil.TestTxnStorage + rpcAgent *testutil.MockRPCAgent + disttaeEngine *testutil.TestDisttaeEngine + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ctx = context.WithValue(ctx, defines.TenantIDKey{}, accountId) + + // mock a schema with 4 columns and the 4th column as primary key + // the first column is the 9th column in the predefined columns in + // the mock function. Here we exepct the type of the primary key + // is types.T_char or types.T_varchar + schema := catalog2.MockSchemaEnhanced(4, primaryKeyIdx, 9) + schema.Name = tableName + + { + opt, err := testutil.GetS3SharedFileServiceOption(ctx, testutil.GetDefaultTestPath("test", t)) + require.NoError(t, err) + + disttaeEngine, taeEngine, rpcAgent, _ = testutil.CreateEngines( + ctx, + testutil.TestOptions{TaeEngineOptions: opt}, + t, + testutil.WithDisttaeEngineWorkspaceThreshold(mpool.MB*2), + testutil.WithDisttaeEngineInsertEntryMaxCount(10000), + ) + defer func() { + disttaeEngine.Close(ctx) + taeEngine.Close(true) + rpcAgent.Close() + }() + + _, _, err = disttaeEngine.CreateDatabaseAndTable(ctx, databaseName, tableName, schema) + require.NoError(t, err) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + } + + testutil2.CompactBlocks(t, 0, taeEngine.GetDB(), databaseName, schema, false) + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 1, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + var blkId *common.ID + for iter.Next() { + if !iter.GetObject().IsAppendable() { + continue + } + blkId = iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + } + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + //start to build sharding readers. + _, rel, txn, err := disttaeEngine.GetTable(ctx, databaseName, tableName) + require.NoError(t, err) + + //relData, err := rel.Ranges(ctx, nil, 0) + //require.NoError(t, err) + shardSvr := testutil.MockShardService() + delegate, _ := disttae.MockTableDelegate(rel, shardSvr) + + relData, err := delegate.Ranges(ctx, nil, 0) + require.NoError(t, err) + + tomb, err := delegate.CollectTombstones(ctx, 0, engine.Policy_CollectAllTombstones) + require.NoError(t, err) + require.True(t, tomb.HasAnyInMemoryTombstone()) + + _, err = delegate.BuildReaders( + ctx, + rel.GetProcess(), + nil, + relData, + 1, + 0, + false, + 0, + ) + require.NoError(t, err) + require.NoError(t, txn.Commit(ctx)) +} + +func Test_ShardingLocalReader(t *testing.T) { + var ( + //err error + mp *mpool.MPool + accountId = catalog.System_Account + tableName = "test_reader_table" + databaseName = "test_reader_database" + + primaryKeyIdx int = 3 + + taeEngine *testutil.TestTxnStorage + rpcAgent *testutil.MockRPCAgent + disttaeEngine *testutil.TestDisttaeEngine + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ctx = context.WithValue(ctx, defines.TenantIDKey{}, accountId) + + // mock a schema with 4 columns and the 4th column as primary key + // the first column is the 9th column in the predefined columns in + // the mock function. Here we exepct the type of the primary key + // is types.T_char or types.T_varchar + schema := catalog2.MockSchemaEnhanced(4, primaryKeyIdx, 9) + schema.Name = tableName + + { + opt, err := testutil.GetS3SharedFileServiceOption(ctx, testutil.GetDefaultTestPath("test", t)) + require.NoError(t, err) + + disttaeEngine, taeEngine, rpcAgent, mp = testutil.CreateEngines( + ctx, + testutil.TestOptions{TaeEngineOptions: opt}, + t, + testutil.WithDisttaeEngineWorkspaceThreshold(mpool.MB*2), + testutil.WithDisttaeEngineInsertEntryMaxCount(10000), + ) + defer func() { + disttaeEngine.Close(ctx) + taeEngine.Close(true) + rpcAgent.Close() + }() + + _, _, err = disttaeEngine.CreateDatabaseAndTable(ctx, databaseName, tableName, schema) + require.NoError(t, err) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + } + + testutil2.CompactBlocks(t, 0, taeEngine.GetDB(), databaseName, schema, false) + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + iter.Next() + blkId := iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + + err := rel.RangeDelete(blkId, 0, 1, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + txn, err := taeEngine.StartTxn() + require.NoError(t, err) + + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + rowsCnt := 10 + bat := catalog2.MockBatch(schema, rowsCnt) + err = rel.Append(ctx, bat) + require.Nil(t, err) + + err = txn.Commit(context.Background()) + require.Nil(t, err) + + } + + { + + txn, _ := taeEngine.StartTxn() + database, _ := txn.GetDatabase(databaseName) + rel, _ := database.GetRelationByName(schema.Name) + + iter := rel.MakeObjectIt(false) + var blkId *common.ID + for iter.Next() { + if !iter.GetObject().IsAppendable() { + continue + } + blkId = iter.GetObject().GetMeta().(*catalog2.ObjectEntry).AsCommonID() + } + err := rel.RangeDelete(blkId, 0, 7, handle.DT_Normal) + require.Nil(t, err) + + require.NoError(t, txn.Commit(context.Background())) + + } + + { + //start to build sharding readers. + _, rel, txn, err := disttaeEngine.GetTable(ctx, databaseName, tableName) + require.NoError(t, err) + + relData, err := rel.Ranges(ctx, nil, 0) + require.NoError(t, err) + + shardSvr := testutil.MockShardService() + delegate, _ := disttae.MockTableDelegate(rel, shardSvr) + num := 10 + rds, err := delegate.BuildShardingReaders( + ctx, + rel.GetProcess(), + nil, + relData, + num, + 0, + false, + 0, + ) + require.NoError(t, err) + + rows := 0 + buildBatch := func() *batch.Batch { + bat := batch.NewWithSize(1) + for _, col := range schema.ColDefs { + if col.Name == schema.ColDefs[primaryKeyIdx].Name { + vec := vector.NewVec(col.Type) + bat.Vecs[0] = vec + bat.Attrs = []string{col.Name} + break + } + } + return bat + } + + for _, r := range rds { + for { + bat := buildBatch() + isEnd, err := r.Read( + ctx, + []string{schema.ColDefs[primaryKeyIdx].Name}, + nil, + mp, + bat, + ) + require.NoError(t, err) + + if isEnd { + break + } + rows += int(bat.RowCount()) + } + } + + require.Equal(t, 2, rows) + + err = txn.Commit(ctx) + require.Nil(t, err) + } + + //test set orderby + shardingLRD := disttae.MockShardingLocalReader() + assert.Panics(t, func() { + shardingLRD.SetOrderBy(nil) + }) + assert.Panics(t, func() { + shardingLRD.GetOrderBy() + }) + assert.Panics(t, func() { + shardingLRD.SetFilterZM(nil) + }) +} diff --git a/pkg/vm/engine/test/testutil/disttae_engine.go b/pkg/vm/engine/test/testutil/disttae_engine.go index 97ef14a3dd93..5ce9894afc9f 100644 --- a/pkg/vm/engine/test/testutil/disttae_engine.go +++ b/pkg/vm/engine/test/testutil/disttae_engine.go @@ -17,10 +17,6 @@ package testutil import ( "context" - "github.com/matrixorigin/matrixone/pkg/logutil" - "github.com/matrixorigin/matrixone/pkg/vm/engine" - catalog2 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" - "strings" "sync" "sync/atomic" @@ -36,18 +32,24 @@ import ( "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/lockservice" "github.com/matrixorigin/matrixone/pkg/logservice" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/lock" logservice2 "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + pb "github.com/matrixorigin/matrixone/pkg/pb/shard" "github.com/matrixorigin/matrixone/pkg/pb/timestamp" qclient "github.com/matrixorigin/matrixone/pkg/queryservice/client" + "github.com/matrixorigin/matrixone/pkg/shardservice" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/compile" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/txn/service" + "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/logtailreplay" + catalog2 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog" + "github.com/matrixorigin/matrixone/pkg/vm/process" ) type TestDisttaeEngine struct { @@ -149,6 +151,18 @@ func NewTestDisttaeEngine( ) runtime.ServiceRuntime("").SetGlobalVariables(runtime.InternalSQLExecutor, sqlExecutor) + runtime.ServiceRuntime("").SetGlobalVariables( + runtime.ProcessCodecService, + process.NewCodecService( + de.txnClient, + fs, + new(mockLockService), + qc, + hakeeper, + nil, + de.Engine, + )) + // InitLoTailPushModel presupposes that the internal sql executor has been initialized. err = de.Engine.InitLogTailPushModel(ctx, de.timestampWaiter) //err = de.prevSubscribeSysTables(ctx, rpcAgent) @@ -518,6 +532,57 @@ func (ml *mockLockService) CloseRemoteLockTable(group uint32, tableID uint64, ve return false, nil } +type mockShardService struct { +} + +func MockShardService() *mockShardService { + return &mockShardService{} +} + +func (ms *mockShardService) Config() shardservice.Config { + return shardservice.Config{Enable: true} +} + +func (ms *mockShardService) GetStorage() shardservice.ShardStorage { + return nil +} + +func (ms *mockShardService) Read(cxt context.Context, req shardservice.ReadRequest, opts shardservice.ReadOptions) error { + return nil +} + +func (ms *mockShardService) HasLocalReplica(tableID, shardID uint64) bool { + return true +} + +func (ms *mockShardService) HasAllLocalReplicas(tableID uint64) bool { + return false +} + +func (ms *mockShardService) GetShardInfo(table uint64) (uint64, pb.Policy, bool, error) { + return table, 1, true, nil +} + +func (ms *mockShardService) Create(ctx context.Context, table uint64, txnOp client.TxnOperator) error { + return nil +} + +func (ms *mockShardService) Delete(ctx context.Context, table uint64, txnOp client.TxnOperator) error { + return nil +} + +func (ms *mockShardService) ReplicaCount() int64 { + return 1 +} + +func (ms *mockShardService) TableReplicaCount(tableID uint64) int64 { + return 1 +} + +func (ms *mockShardService) Close() error { + return nil +} + var _ logservice.CNHAKeeperClient = new(testHAKeeperClient) type testHAKeeperClient struct { diff --git a/pkg/vm/engine/types.go b/pkg/vm/engine/types.go index da79b7d40b86..d48554934913 100644 --- a/pkg/vm/engine/types.go +++ b/pkg/vm/engine/types.go @@ -583,6 +583,14 @@ const ( TombstoneData ) +type TombstoneCollectPolicy uint64 + +const ( + Policy_CollectUncommittedTombstones = 1 << iota + Policy_CollectCommittedTombstones + Policy_CollectAllTombstones = Policy_CollectUncommittedTombstones | Policy_CollectCommittedTombstones +) + type TombstoneApplyPolicy uint64 const ( @@ -728,6 +736,14 @@ const ( End ) +type DataSourceType uint8 + +const ( + GeneralLocalDataSource DataSourceType = iota + ShardingLocalDataSource + ShardingRemoteDataSource +) + type DataSource interface { Next( ctx context.Context, @@ -789,6 +805,7 @@ type ChangesHandle interface { Next(ctx context.Context, mp *mpool.MPool) (data *batch.Batch, tombstone *batch.Batch, hint ChangesHandle_Hint, err error) Close() error } + type Relation interface { Statistics @@ -798,8 +815,10 @@ type Relation interface { // third parameter: Transaction offset used to specify the starting position for reading data. Ranges(context.Context, []*plan.Expr, int) (RelData, error) - CollectTombstones(ctx context.Context, txnOffset int) (Tombstoner, error) + CollectTombstones(ctx context.Context, txnOffset int, policy TombstoneCollectPolicy) (Tombstoner, error) + CollectChanges(ctx context.Context, from, to types.TS, mp *mpool.MPool) (ChangesHandle, error) + TableDefs(context.Context) ([]TableDef, error) // Get complete tableDef information, including columns, constraints, partitions, version, comments, etc @@ -846,6 +865,17 @@ type Relation interface { policy TombstoneApplyPolicy, ) ([]Reader, error) + BuildShardingReaders( + ctx context.Context, + proc any, + expr *plan.Expr, + relData RelData, + num int, + txnOffset int, + orderBy bool, + policy TombstoneApplyPolicy, + ) ([]Reader, error) + TableColumns(ctx context.Context) ([]*Attribute, error) //max and min values @@ -853,6 +883,8 @@ type Relation interface { GetEngineType() EngineType + GetProcess() any + GetColumMetadataScanInfo(ctx context.Context, name string) ([]*plan.MetadataScanInfo, error) // PrimaryKeysMayBeModified reports whether any rows with any primary keys in keyVector was modified during `from` to `to` diff --git a/pkg/vm/process/process_codec.go b/pkg/vm/process/process_codec.go index 1d9dcddb1ca4..10748f00662a 100644 --- a/pkg/vm/process/process_codec.go +++ b/pkg/vm/process/process_codec.go @@ -18,6 +18,9 @@ import ( "context" "time" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -31,10 +34,17 @@ import ( "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/udf" "github.com/matrixorigin/matrixone/pkg/vm/engine" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) +func MockProcessInfoWithPro( + sql string, + pro any, +) (pipeline.ProcessInfo, error) { + process := pro.(*Process) + process.Base.SessionInfo.TimeZone = time.UTC + return process.BuildProcessInfo(sql) +} + func (proc *Process) BuildProcessInfo( sql string, ) (pipeline.ProcessInfo, error) { @@ -69,9 +79,13 @@ func (proc *Process) BuildProcessInfo( } } { // session info - timeBytes, err := time.Time{}.In(proc.Base.SessionInfo.TimeZone).MarshalBinary() - if err != nil { - return procInfo, err + var timeBytes []byte + var err error + if proc.Base.SessionInfo.TimeZone != nil { + timeBytes, err = time.Time{}.In(proc.Base.SessionInfo.TimeZone).MarshalBinary() + if err != nil { + return procInfo, err + } } procInfo.SessionInfo = pipeline.SessionInfo{ diff --git a/proto/shard.proto b/proto/shard.proto index b8ba7809a67f..3c76182dcb8b 100644 --- a/proto/shard.proto +++ b/proto/shard.proto @@ -229,8 +229,11 @@ message ReadParam { GetColumMetadataScanInfoParam GetColumMetadataScanInfoParam = 6 [(gogoproto.nullable) = false]; MergeObjectsParam MergeObjectsParam = 7 [(gogoproto.nullable) = false]; KeyParam KeyParam = 8 [(gogoproto.nullable) = false]; - ReaderParam ReaderParam = 9 [(gogoproto.nullable) = false]; + ReaderBuildParam ReaderBuildParam = 9 [(gogoproto.nullable) = false]; PrimaryKeysMayBeModifiedParam PrimaryKeysMayBeModifiedParam = 10 [(gogoproto.nullable) = false]; + ReadNextParam ReadNextParam = 11 [(gogoproto.nullable) = false]; + ReadCloseParam ReadCloseParam = 12 [(gogoproto.nullable) = false]; + CollectTombstonesParam CollectTombstonesParam = 13 [(gogoproto.nullable) = false]; } message TxnTable { @@ -253,6 +256,10 @@ message RangesParam { bytes UncommittedObjects = 2; } +message CollectTombstonesParam { + uint64 collectPolicy = 1; +} + message GetColumMetadataScanInfoParam { string columnName = 1; } @@ -261,12 +268,28 @@ message KeyParam { bytes Key = 1; } -message ReaderParam { - int32 num = 1; - plan.Expr expr = 2 [(gogoproto.nullable) = false]; - bytes ranges = 3; - bool orderedScan = 4; - int32 txnOffset = 5; +message ReaderBuildParam { + bytes relData = 1; + plan.Expr expr = 2; + int32 scanType = 3; + int32 tombstoneApplyPolicy = 4; +} + +message ReaderBuildResult { + bytes uuid = 1; +} + +message ReadNextParam { + bytes uuid = 1; + repeated string columns = 2; +} + +message ReadNextResult { + bytes result = 1; +} + +message ReadCloseParam { + bytes uuid = 1; } message PrimaryKeysMayBeModifiedParam { diff --git a/test/distributed/cases/ddl/partition_prune.result b/test/distributed/cases/ddl/partition_prune.result index 80173679a201..b766c9f694b7 100644 --- a/test/distributed/cases/ddl/partition_prune.result +++ b/test/distributed/cases/ddl/partition_prune.result @@ -50,8 +50,8 @@ col1 col2 col3 select * from t1 where col3 in (7990, 7698, 7988); col1 col2 col3 23 2001-12-04 7990 -6 1981-05-01 7698 21 1771-10-06 7988 +6 1981-05-01 7698 select * from t1 where col3 = 7996 and col1 > 25; col1 col2 col3 29 2001-11-17 7996 @@ -289,15 +289,15 @@ id fname lname hired separated job_code store_id 10025 Prasadram Heyers 1958-10-31 1987-08-17 180 8 select * from employees where store_id = 8 or store_id = 10; id fname lname hired separated job_code store_id -10011 Mary Sluis 1953-11-07 1990-01-22 120 8 -10015 Guoxiang Nooteboom 1959-08-19 1987-07-02 140 8 -10025 Prasadram Heyers 1958-10-31 1987-08-17 180 8 10008 Saniya Kalloufi 1958-02-19 1994-09-15 170 10 10010 Duangkaew Piveteau 1963-06-01 1989-08-24 160 10 10017 Cristinel Bouloucos 1958-07-06 1993-08-03 170 10 10022 Shahaf Famili 1952-07-08 1995-08-22 130 10 10027 Divier Reistad 1962-07-10 1989-07-07 180 10 10031 Karsten Joslin 1959-01-27 1991-09-01 110 10 +10011 Mary Sluis 1953-11-07 1990-01-22 120 8 +10015 Guoxiang Nooteboom 1959-08-19 1987-07-02 140 8 +10025 Prasadram Heyers 1958-10-31 1987-08-17 180 8 select * from employees where store_id in (1, 2, 11); id fname lname hired separated job_code store_id 10001 Georgi Facello 1953-09-02 1986-06-26 120 1 @@ -308,28 +308,28 @@ id fname lname hired separated job_code store_id 10030 Elvis Demeyer 1958-07-14 1994-02-17 110 1 select * from employees where store_id in (1, 2, 6, 7); id fname lname hired separated job_code store_id +10002 Bezalel Simmel 1964-06-02 1985-11-21 150 7 +10007 Tzvetan Zielinski 1957-05-23 1989-02-10 110 6 +10012 Patricio Bridgland 1960-10-04 1992-12-18 120 7 +10038 Huan Lortz 1960-07-20 1989-09-20 140 7 10001 Georgi Facello 1953-09-02 1986-06-26 120 1 10016 Kazuhito Cappelletti 1961-05-02 1995-01-27 140 2 10018 Kazuhide Peha 1954-06-19 1987-04-03 170 2 10020 Mayuko Warwick 1952-12-24 1991-01-26 120 1 10030 Elvis Demeyer 1958-07-14 1994-02-17 110 1 +select * from employees where store_id in (1, 2, 11) or store_id in (6, 7, 18); +id fname lname hired separated job_code store_id 10002 Bezalel Simmel 1964-06-02 1985-11-21 150 7 +10005 Kyoichi Maliniak 1955-01-21 1989-09-12 150 18 10007 Tzvetan Zielinski 1957-05-23 1989-02-10 110 6 10012 Patricio Bridgland 1960-10-04 1992-12-18 120 7 10038 Huan Lortz 1960-07-20 1989-09-20 140 7 -select * from employees where store_id in (1, 2, 11) or store_id in (6, 7, 18); -id fname lname hired separated job_code store_id 10001 Georgi Facello 1953-09-02 1986-06-26 120 1 10016 Kazuhito Cappelletti 1961-05-02 1995-01-27 140 2 10018 Kazuhide Peha 1954-06-19 1987-04-03 170 2 10020 Mayuko Warwick 1952-12-24 1991-01-26 120 1 10028 Domenick Tempesti 1963-11-26 1991-10-22 110 11 10030 Elvis Demeyer 1958-07-14 1994-02-17 110 1 -10002 Bezalel Simmel 1964-06-02 1985-11-21 150 7 -10005 Kyoichi Maliniak 1955-01-21 1989-09-12 150 18 -10007 Tzvetan Zielinski 1957-05-23 1989-02-10 110 6 -10012 Patricio Bridgland 1960-10-04 1992-12-18 120 7 -10038 Huan Lortz 1960-07-20 1989-09-20 140 7 select * from employees where store_id = 3 and id = 10004 or store_id = 10; id fname lname hired separated job_code store_id 10004 Chirstian Koblick 1954-05-01 1986-12-01 150 3 @@ -360,4 +360,66 @@ id fname lname hired separated job_code store_id 10027 Divier Reistad 1962-07-10 1989-07-07 180 10 10031 Karsten Joslin 1959-01-27 1991-09-01 110 10 10004 Chirstian Koblick 1954-05-01 1986-12-01 150 3 +drop table if exists employees; +CREATE TABLE employees ( +id INT NOT NULL, +fname VARCHAR(30), +lname VARCHAR(30), +hired DATE NOT NULL DEFAULT '1970-01-01', +separated DATE NOT NULL DEFAULT '9999-12-31', +job_code INT, +store_id INT +) PARTITION BY HASH(store_id) PARTITIONS 4; +INSERT INTO employees VALUES +(10001, 'Georgi', 'Facello', '1953-09-02','1986-06-26',120, 1), +(10002, 'Bezalel', 'Simmel', '1964-06-02','1985-11-21',150, 7), +(10003, 'Parto', 'Bamford', '1959-12-03','1986-08-28',140, 3), +(10004, 'Chirstian', 'Koblick', '1954-05-01','1986-12-01',150, 3), +(10005, 'Kyoichi', 'Maliniak', '1955-01-21','1989-09-12',150, 18), +(10006, 'Anneke', 'Preusig', '1953-04-20','1989-06-02',150, 15), +(10007, 'Tzvetan', 'Zielinski', '1957-05-23','1989-02-10',110, 6), +(10008, 'Saniya', 'Kalloufi', '1958-02-19','1994-09-15',170, 10), +(10009, 'Sumant', 'Peac', '1952-04-19','1985-02-18',110, 13), +(10010, 'Duangkaew', 'Piveteau', '1963-06-01','1989-08-24',160, 10), +(10011, 'Mary', 'Sluis', '1953-11-07','1990-01-22',120, 8), +(10012, 'Patricio', 'Bridgland', '1960-10-04','1992-12-18',120, 7), +(10013, 'Eberhardt', 'Terkki', '1963-06-07','1985-10-20',160, 17), +(10014, 'Berni', 'Genin', '1956-02-12','1987-03-11',120, 15), +(10015, 'Guoxiang', 'Nooteboom', '1959-08-19','1987-07-02',140, 8), +(10016, 'Kazuhito', 'Cappelletti', '1961-05-02','1995-01-27',140, 2), +(10017, 'Cristinel', 'Bouloucos', '1958-07-06','1993-08-03',170, 10), +(10018, 'Kazuhide', 'Peha', '1954-06-19','1987-04-03',170, 2), +(10019, 'Lillian', 'Haddadi', '1953-01-23','1999-04-30',170, 13), +(10020, 'Mayuko', 'Warwick', '1952-12-24','1991-01-26',120, 1), +(10021, 'Ramzi', 'Erde', '1960-02-20','1988-02-10',120, 9), +(10022, 'Shahaf', 'Famili', '1952-07-08','1995-08-22',130, 10), +(10023, 'Bojan', 'Montemayor', '1953-09-29','1989-12-17',120, 5), +(10024, 'Suzette', 'Pettey', '1958-09-05','1997-05-19',130, 4), +(10025, 'Prasadram', 'Heyers', '1958-10-31','1987-08-17',180, 8), +(10026, 'Yongqiao', 'Berztiss', '1953-04-03','1995-03-20',170, 4), +(10027, 'Divier', 'Reistad', '1962-07-10','1989-07-07',180, 10), +(10028, 'Domenick', 'Tempesti', '1963-11-26','1991-10-22',110, 11), +(10029, 'Otmar', 'Herbst', '1956-12-13','1985-11-20',110, 12), +(10030, 'Elvis', 'Demeyer', '1958-07-14','1994-02-17',110, 1), +(10031, 'Karsten', 'Joslin', '1959-01-27','1991-09-01',110, 10), +(10032, 'Jeong', 'Reistad', '1960-08-09','1990-06-20',120, 19), +(10033, 'Arif', 'Merlo', '1956-11-14','1987-03-18',120, 14), +(10034, 'Bader', 'Swan', '1962-12-29','1988-09-21',130, 16), +(10035, 'Alain', 'Chappelet', '1953-02-08','1988-09-05',130, 3), +(10036, 'Adamantios', 'Portugali', '1959-08-10','1992-01-03',130, 14), +(10037, 'Pradeep', 'Makrucki', '1963-07-22','1990-12-05',140, 12), +(10038, 'Huan', 'Lortz', '1960-07-20','1989-09-20',140, 7), +(10039, 'Alejandro', 'Brender', '1959-10-01','1988-01-19',110, 20), +(10040, 'Weiyi', 'Meriste', '1959-09-13','1993-02-14',140, 17); +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +delete from employees where store_id = (select min(store_id) from employees); drop database db1; diff --git a/test/distributed/cases/ddl/partition_prune.sql b/test/distributed/cases/ddl/partition_prune.sql index cb0a00aa7efd..5c9b3aef8333 100644 --- a/test/distributed/cases/ddl/partition_prune.sql +++ b/test/distributed/cases/ddl/partition_prune.sql @@ -205,4 +205,71 @@ select * from employees where store_id > 15; --无分区裁剪 select * from employees where store_id = 10 or id = 10004; +-------------------------------------------------------------------------------------- +drop table if exists employees; +CREATE TABLE employees ( + id INT NOT NULL, + fname VARCHAR(30), + lname VARCHAR(30), + hired DATE NOT NULL DEFAULT '1970-01-01', + separated DATE NOT NULL DEFAULT '9999-12-31', + job_code INT, + store_id INT +) PARTITION BY HASH(store_id) PARTITIONS 4; + + +INSERT INTO employees VALUES + (10001, 'Georgi', 'Facello', '1953-09-02','1986-06-26',120, 1), + (10002, 'Bezalel', 'Simmel', '1964-06-02','1985-11-21',150, 7), + (10003, 'Parto', 'Bamford', '1959-12-03','1986-08-28',140, 3), + (10004, 'Chirstian', 'Koblick', '1954-05-01','1986-12-01',150, 3), + (10005, 'Kyoichi', 'Maliniak', '1955-01-21','1989-09-12',150, 18), + (10006, 'Anneke', 'Preusig', '1953-04-20','1989-06-02',150, 15), + (10007, 'Tzvetan', 'Zielinski', '1957-05-23','1989-02-10',110, 6), + (10008, 'Saniya', 'Kalloufi', '1958-02-19','1994-09-15',170, 10), + (10009, 'Sumant', 'Peac', '1952-04-19','1985-02-18',110, 13), + (10010, 'Duangkaew', 'Piveteau', '1963-06-01','1989-08-24',160, 10), + (10011, 'Mary', 'Sluis', '1953-11-07','1990-01-22',120, 8), + (10012, 'Patricio', 'Bridgland', '1960-10-04','1992-12-18',120, 7), + (10013, 'Eberhardt', 'Terkki', '1963-06-07','1985-10-20',160, 17), + (10014, 'Berni', 'Genin', '1956-02-12','1987-03-11',120, 15), + (10015, 'Guoxiang', 'Nooteboom', '1959-08-19','1987-07-02',140, 8), + (10016, 'Kazuhito', 'Cappelletti', '1961-05-02','1995-01-27',140, 2), + (10017, 'Cristinel', 'Bouloucos', '1958-07-06','1993-08-03',170, 10), + (10018, 'Kazuhide', 'Peha', '1954-06-19','1987-04-03',170, 2), + (10019, 'Lillian', 'Haddadi', '1953-01-23','1999-04-30',170, 13), + (10020, 'Mayuko', 'Warwick', '1952-12-24','1991-01-26',120, 1), + (10021, 'Ramzi', 'Erde', '1960-02-20','1988-02-10',120, 9), + (10022, 'Shahaf', 'Famili', '1952-07-08','1995-08-22',130, 10), + (10023, 'Bojan', 'Montemayor', '1953-09-29','1989-12-17',120, 5), + (10024, 'Suzette', 'Pettey', '1958-09-05','1997-05-19',130, 4), + (10025, 'Prasadram', 'Heyers', '1958-10-31','1987-08-17',180, 8), + (10026, 'Yongqiao', 'Berztiss', '1953-04-03','1995-03-20',170, 4), + (10027, 'Divier', 'Reistad', '1962-07-10','1989-07-07',180, 10), + (10028, 'Domenick', 'Tempesti', '1963-11-26','1991-10-22',110, 11), + (10029, 'Otmar', 'Herbst', '1956-12-13','1985-11-20',110, 12), + (10030, 'Elvis', 'Demeyer', '1958-07-14','1994-02-17',110, 1), + (10031, 'Karsten', 'Joslin', '1959-01-27','1991-09-01',110, 10), + (10032, 'Jeong', 'Reistad', '1960-08-09','1990-06-20',120, 19), + (10033, 'Arif', 'Merlo', '1956-11-14','1987-03-18',120, 14), + (10034, 'Bader', 'Swan', '1962-12-29','1988-09-21',130, 16), + (10035, 'Alain', 'Chappelet', '1953-02-08','1988-09-05',130, 3), + (10036, 'Adamantios', 'Portugali', '1959-08-10','1992-01-03',130, 14), + (10037, 'Pradeep', 'Makrucki', '1963-07-22','1990-12-05',140, 12), + (10038, 'Huan', 'Lortz', '1960-07-20','1989-09-20',140, 7), + (10039, 'Alejandro', 'Brender', '1959-10-01','1988-01-19',110, 20), + (10040, 'Weiyi', 'Meriste', '1959-09-13','1993-02-14',140, 17); + +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +insert into employees select * from employees; +delete from employees where store_id = (select min(store_id) from employees); + drop database db1; \ No newline at end of file