Skip to content

Commit

Permalink
[Feature]Sharding support 1 (#18533)
Browse files Browse the repository at this point in the history
1.  Finish collecting tombstones from remote CN.
2. Handle sharding reader

Todo:
- [ ] Handle order by
- [ ] Improve the performance for sharding.

Approved by: @daviszhen, @badboynt1, @zhangxu19830126, @m-schen, @reusee, @ouyuanning, @qingxinhome, @aunjgr, @heni02, @aressu1985, @iamlinjunhong, @XuPeng-SH
  • Loading branch information
triump2020 committed Sep 15, 2024
1 parent aa44acd commit 34ef42a
Show file tree
Hide file tree
Showing 25 changed files with 3,603 additions and 499 deletions.
8 changes: 6 additions & 2 deletions pkg/cnservice/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (

"github.com/fagongzi/goetty/v2"
"github.com/google/uuid"
"go.uber.org/zap"

"github.com/matrixorigin/matrixone/pkg/bootstrap"
"github.com/matrixorigin/matrixone/pkg/catalog"
"github.com/matrixorigin/matrixone/pkg/clusterservice"
Expand Down Expand Up @@ -66,7 +68,6 @@ import (
"github.com/matrixorigin/matrixone/pkg/vm/engine/disttae"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/blockio"
"github.com/matrixorigin/matrixone/pkg/vm/process"
"go.uber.org/zap"
)

func NewService(
Expand Down Expand Up @@ -764,10 +765,13 @@ func (s *service) initShardService() {
shardservice.ReadApproxObjectsNum: disttae.HandleShardingReadApproxObjectsNum,
shardservice.ReadRanges: disttae.HandleShardingReadRanges,
shardservice.ReadGetColumMetadataScanInfo: disttae.HandleShardingReadGetColumMetadataScanInfo,
shardservice.ReadReader: disttae.HandleShardingReadReader,
shardservice.ReadBuildReader: disttae.HandleShardingReadBuildReader,
shardservice.ReadPrimaryKeysMayBeModified: disttae.HandleShardingReadPrimaryKeysMayBeModified,
shardservice.ReadMergeObjects: disttae.HandleShardingReadMergeObjects,
shardservice.ReadVisibleObjectStats: disttae.HandleShardingReadVisibleObjectStats,
shardservice.ReadClose: disttae.HandleShardingReadClose,
shardservice.ReadNext: disttae.HandleShardingReadNext,
shardservice.ReadCollectTombstones: disttae.HandleShardingReadCollectTombstones,
},
s.storeEngine,
)
Expand Down
4 changes: 4 additions & 0 deletions pkg/common/morpc/buffer_pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package morpc

import (
"github.com/fagongzi/goetty/v2/buf"

"github.com/matrixorigin/matrixone/pkg/common/reuse"
)

Expand Down Expand Up @@ -91,5 +92,8 @@ func (b *Buffer) Mark() {
}

func (b *Buffer) GetMarkedData() []byte {
if b.markIdx == b.buf.GetWriteIndex() {
return nil
}
return b.buf.RawSlice(b.markIdx, b.buf.GetWriteIndex())
}
136 changes: 115 additions & 21 deletions pkg/frontend/test/engine_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 34ef42a

Please sign in to comment.