Merge pull request #8 from mo3et/version-sync

feat: implement List PaginationV2, update use latest errs pkg and remove unnecessary data transfer.
withchao · Jul 1, 2024 · b71e4ff · b71e4ff
2 parents e06a676 + 55c9915
commit b71e4ff
Show file tree

Hide file tree

Showing 26 changed files with 500 additions and 296 deletions.
diff --git a/internal/friend/sdk.go b/internal/friend/sdk.go
@@ -238,6 +238,58 @@ func (f *Friend) GetFriendListPage(ctx context.Context, offset, count int32) ([]
 	return res, nil
 }
 
+func (f *Friend) GetFriendListPageV2(ctx context.Context, offset, count int32) (*GetFriendInfoListV2, error) {
+	datafetcher := datafetcher.NewDataFetcher(
+		f.db,
+		f.friendListTableName(),
+		f.loginUserID,
+		func(localFriend *model_struct.LocalFriend) string {
+			return localFriend.FriendUserID
+		},
+		func(ctx context.Context, values []*model_struct.LocalFriend) error {
+			return f.db.BatchInsertFriend(ctx, values)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalFriend, error) {
+			return f.db.GetFriendInfoList(ctx, userIDs)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalFriend, error) {
+			serverFriend, err := f.GetDesignatedFriends(ctx, userIDs)
+			if err != nil {
+				return nil, err
+			}
+			return datautil.Batch(ServerFriendToLocalFriend, serverFriend), nil
+		},
+	)
+
+	localFriendList, isEnd, err := datafetcher.FetchWithPaginationV2(ctx, int(offset), int(count))
+	if err != nil {
+		return nil, err
+	}
+
+	// don't need extra handle. only full pull.
+	localBlackList, err := f.db.GetBlackListDB(ctx)
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]*model_struct.LocalBlack)
+	for i, black := range localBlackList {
+		m[black.BlockUserID] = localBlackList[i]
+	}
+	fullUserInfo := make([]*server_api_params.FullUserInfo, 0, len(localFriendList))
+	for _, localFriend := range localFriendList {
+		fullUserInfo = append(fullUserInfo, &server_api_params.FullUserInfo{
+			PublicInfo: nil,
+			FriendInfo: localFriend,
+			BlackInfo:  m[localFriend.FriendUserID],
+		})
+	}
+	response := &GetFriendInfoListV2{
+		FullUserInfoList: fullUserInfo,
+		IsEnd:            isEnd,
+	}
+	return response, nil
+}
+
 func (f *Friend) SearchFriends(ctx context.Context, param *sdk.SearchFriendsParam) ([]*sdk.SearchFriendItem, error) {
 	if len(param.KeywordList) == 0 || (!param.IsSearchNickname && !param.IsSearchUserID && !param.IsSearchRemark) {
 		return nil, sdkerrs.ErrArgs.WrapMsg("keyword is null or search field all false")

diff --git a/internal/friend/types.go b/internal/friend/types.go
@@ -0,0 +1,8 @@
+package friend
+
+import "github.com/openimsdk/openim-sdk-core/v3/pkg/server_api_params"
+
+type GetFriendInfoListV2 struct {
+	FullUserInfoList []*server_api_params.FullUserInfo
+	IsEnd            bool `json:"isEnd"`
+}
diff --git a/internal/group/sdk.go b/internal/group/sdk.go
@@ -212,6 +212,41 @@ func (g *Group) GetSpecifiedGroupsInfo(ctx context.Context, groupIDs []string) (
 	return dataFetcher.FetchMissingAndCombineLocal(ctx, groupIDs)
 }
 
+func (g *Group) GetJoinedGroupListPageV2(ctx context.Context, offset, count int32) (*GetGroupListV2Response, error) {
+	dataFetcher := datafetcher.NewDataFetcher(
+		g.db,
+		g.groupTableName(),
+		g.loginUserID,
+		func(localGroup *model_struct.LocalGroup) string {
+			return localGroup.GroupID
+		},
+		func(ctx context.Context, values []*model_struct.LocalGroup) error {
+			return g.db.BatchInsertGroup(ctx, values)
+		},
+		func(ctx context.Context, groupIDs []string) ([]*model_struct.LocalGroup, error) {
+			return g.db.GetGroups(ctx, groupIDs)
+		},
+		func(ctx context.Context, groupIDs []string) ([]*model_struct.LocalGroup, error) {
+			serverGroupInfo, err := g.getGroupsInfoFromSvr(ctx, groupIDs)
+			if err != nil {
+				return nil, err
+			}
+			return datautil.Batch(ServerGroupToLocalGroup, serverGroupInfo), nil
+		},
+	)
+
+	groupsList, isEnd, err := dataFetcher.FetchWithPaginationV2(ctx, int(offset), int(count))
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &GetGroupListV2Response{
+		GroupsList: groupsList,
+		IsEnd:      isEnd,
+	}
+	return resp, nil
+}
+
 func (g *Group) SearchGroups(ctx context.Context, param sdk_params_callback.SearchGroupsParam) ([]*model_struct.LocalGroup, error) {
 	if len(param.KeywordList) == 0 || (!param.IsSearchGroupName && !param.IsSearchGroupID) {
 		return nil, sdkerrs.ErrArgs.WrapMsg("keyword is null or search field all false")
@@ -243,7 +278,68 @@ func (g *Group) GetGroupMemberListByJoinTimeFilter(ctx context.Context, groupID
 	if joinTimeEnd == 0 {
 		joinTimeEnd = time.Now().UnixMilli()
 	}
-	return g.db.GetGroupMemberListSplitByJoinTimeFilter(ctx, groupID, int(offset), int(count), joinTimeBegin, joinTimeEnd, userIDs)
+
+	dataFetcher := datafetcher.NewDataFetcher(
+		g.db,
+		g.groupAndMemberVersionTableName(),
+		groupID,
+		func(localGroupMember *model_struct.LocalGroupMember) string {
+			return localGroupMember.UserID
+		},
+		func(ctx context.Context, values []*model_struct.LocalGroupMember) error {
+			return g.db.BatchInsertGroupMember(ctx, values)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			return g.db.GetGroupMemberListSplitByJoinTimeFilter(ctx, groupID, int(offset), int(count), joinTimeBegin, joinTimeEnd, userIDs)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			serverGroupMember, err := g.GetDesignatedGroupMembers(ctx, groupID, userIDs)
+			if err != nil {
+				return nil, err
+			}
+			return datautil.Batch(ServerGroupMemberToLocalGroupMember, serverGroupMember), nil
+		},
+	)
+
+	return dataFetcher.FetchWithPagination(ctx, int(offset), int(count))
+}
+
+func (g *Group) GetGroupMemberListByJoinTimeFilterV2(ctx context.Context, groupID string, offset, count int32, joinTimeBegin, joinTimeEnd int64, userIDs []string) (*GetGroupMemberListV2Response, error) {
+	if joinTimeEnd == 0 {
+		joinTimeEnd = time.Now().UnixMilli()
+	}
+
+	dataFetcher := datafetcher.NewDataFetcher(
+		g.db,
+		g.groupAndMemberVersionTableName(),
+		groupID,
+		func(localGroupMember *model_struct.LocalGroupMember) string {
+			return localGroupMember.UserID
+		},
+		func(ctx context.Context, values []*model_struct.LocalGroupMember) error {
+			return g.db.BatchInsertGroupMember(ctx, values)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			return g.db.GetGroupMemberListSplitByJoinTimeFilter(ctx, groupID, int(offset), int(count), joinTimeBegin, joinTimeEnd, userIDs)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			serverGroupMember, err := g.GetDesignatedGroupMembers(ctx, groupID, userIDs)
+			if err != nil {
+				return nil, err
+			}
+			return datautil.Batch(ServerGroupMemberToLocalGroupMember, serverGroupMember), nil
+		},
+	)
+
+	groupMembersList, isEnd, err := dataFetcher.FetchWithPaginationV2(ctx, int(offset), int(count))
+	if err != nil {
+		return nil, err
+	}
+	resp := &GetGroupMemberListV2Response{
+		GroupMembersList: groupMembersList,
+		IsEnd:            isEnd,
+	}
+	return resp, nil
 }
 
 func (g *Group) GetSpecifiedGroupMembersInfo(ctx context.Context, groupID string, userIDList []string) ([]*model_struct.LocalGroupMember, error) {
@@ -291,7 +387,6 @@ func (g *Group) GetSpecifiedGroupMembersInfo(ctx context.Context, groupID string
 		},
 	)
 	return dataFetcher.FetchMissingAndFillLocal(ctx, userIDList)
-	// return g.db.GetGroupSomeMemberInfo(ctx, groupID, userIDList)
 }
 
 func (g *Group) GetGroupMemberList(ctx context.Context, groupID string, filter, offset, count int32) ([]*model_struct.LocalGroupMember, error) {
@@ -339,6 +434,40 @@ func (g *Group) GetGroupMemberList(ctx context.Context, groupID string, filter,
 	return dataFetcher.FetchWithPagination(ctx, int(offset), int(count))
 }
 
+func (g *Group) GetGroupMemberListV2(ctx context.Context, groupID string, filter, offset, count int32) (*GetGroupMemberListV2Response, error) {
+	dataFetcher := datafetcher.NewDataFetcher(
+		g.db,
+		g.groupAndMemberVersionTableName(),
+		groupID,
+		func(localGroupMember *model_struct.LocalGroupMember) string {
+			return localGroupMember.UserID
+		},
+		func(ctx context.Context, values []*model_struct.LocalGroupMember) error {
+			return g.db.BatchInsertGroupMember(ctx, values)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			return g.db.GetGroupMemberListByUserIDs(ctx, groupID, filter, userIDs)
+		},
+		func(ctx context.Context, userIDs []string) ([]*model_struct.LocalGroupMember, error) {
+			serverGroupMember, err := g.GetDesignatedGroupMembers(ctx, groupID, userIDs)
+			if err != nil {
+				return nil, err
+			}
+			return datautil.Batch(ServerGroupMemberToLocalGroupMember, serverGroupMember), nil
+		},
+	)
+	groupMembersList, isEnd, err := dataFetcher.FetchWithPaginationV2(ctx, int(offset), int(count))
+	if err != nil {
+		return nil, err
+	}
+	resp := &GetGroupMemberListV2Response{
+		GroupMembersList: groupMembersList,
+		IsEnd:            isEnd,
+	}
+
+	return resp, nil
+}
+
 func (g *Group) GetGroupApplicationListAsRecipient(ctx context.Context) ([]*model_struct.LocalAdminGroupRequest, error) {
 	return g.db.GetAdminGroupApplication(ctx)
 }

diff --git a/internal/group/types.go b/internal/group/types.go
@@ -0,0 +1,13 @@
+package group
+
+import "github.com/openimsdk/openim-sdk-core/v3/pkg/db/model_struct"
+
+type GetGroupMemberListV2Response struct {
+	GroupMembersList []*model_struct.LocalGroupMember
+	IsEnd            bool `json:"isEnd"`
+}
+
+type GetGroupListV2Response struct {
+	GroupsList []*model_struct.LocalGroup
+	IsEnd      bool `json:"isEnd"`
+}
diff --git a/open_im_sdk/friend.go b/open_im_sdk/friend.go
@@ -28,6 +28,10 @@ func GetFriendListPage(callback open_im_sdk_callback.Base, operationID string, o
 	call(callback, operationID, UserForSDK.Friend().GetFriendListPage, offset, count)
 }
 
+func GetFriendListPageV2(callback open_im_sdk_callback.Base, operationID string, offset int32, count int32) {
+	call(callback, operationID, UserForSDK.Friend().GetFriendListPageV2, offset, count)
+}
+
 func SearchFriends(callback open_im_sdk_callback.Base, operationID string, searchParam string) {
 	call(callback, operationID, UserForSDK.Friend().SearchFriends, searchParam)
 }

diff --git a/open_im_sdk/group.go b/open_im_sdk/group.go
@@ -88,6 +88,10 @@ func GetJoinedGroupListPage(callback open_im_sdk_callback.Base, operationID stri
 	call(callback, operationID, UserForSDK.Group().GetJoinedGroupListPage, offset, count)
 }
 
+func GetJoinedGroupListPageV2(callback open_im_sdk_callback.Base, operationID string, offset, count int32) {
+	call(callback, operationID, UserForSDK.Group().GetJoinedGroupListPageV2, offset, count)
+}
+
 func GetSpecifiedGroupsInfo(callback open_im_sdk_callback.Base, operationID string, groupIDList string) {
 	call(callback, operationID, UserForSDK.Group().GetSpecifiedGroupsInfo, groupIDList)
 }
@@ -104,6 +108,10 @@ func GetGroupMemberListByJoinTimeFilter(callback open_im_sdk_callback.Base, oper
 	call(callback, operationID, UserForSDK.Group().GetGroupMemberListByJoinTimeFilter, groupID, offset, count, joinTimeBegin, joinTimeEnd, filterUserIDList)
 }
 
+func GetGroupMemberListByJoinTimeFilterV2(callback open_im_sdk_callback.Base, operationID string, groupID string, offset int32, count int32, joinTimeBegin int64, joinTimeEnd int64, filterUserIDList string) {
+	call(callback, operationID, UserForSDK.Group().GetGroupMemberListByJoinTimeFilterV2, groupID, offset, count, joinTimeBegin, joinTimeEnd, filterUserIDList)
+}
+
 func GetSpecifiedGroupMembersInfo(callback open_im_sdk_callback.Base, operationID string, groupID string, userIDList string) {
 	call(callback, operationID, UserForSDK.Group().GetSpecifiedGroupMembersInfo, groupID, userIDList)
 }
@@ -112,6 +120,10 @@ func GetGroupMemberList(callback open_im_sdk_callback.Base, operationID string,
 	call(callback, operationID, UserForSDK.Group().GetGroupMemberList, groupID, filter, offset, count)
 }
 
+func GetGroupMemberListV2(callback open_im_sdk_callback.Base, operationID string, groupID string, filter int32, offset int32, count int32) {
+	call(callback, operationID, UserForSDK.Group().GetGroupMemberListV2, groupID, filter, offset, count)
+}
+
 func GetGroupApplicationListAsRecipient(callback open_im_sdk_callback.Base, operationID string) {
 	call(callback, operationID, UserForSDK.Group().GetGroupApplicationListAsRecipient)
 }

diff --git a/pkg/datafetcher/datafetcher.go b/pkg/datafetcher/datafetcher.go
@@ -126,3 +126,60 @@ func (ds *DataFetcher[T]) FetchMissingAndCombineLocal(ctx context.Context, uids
 
 	return localData, nil
 }
+
+func (ds *DataFetcher[T]) FetchWithPaginationV2(ctx context.Context, offset, limit int) ([]T, bool, error) {
+	var isEnd bool
+	versionInfo, err := ds.db.GetVersionSync(ctx, ds.TableName, ds.EntityID)
+	if err != nil {
+		return nil, isEnd, err
+	}
+
+	if offset > len(versionInfo.UIDList) {
+		return nil, isEnd, errs.New("offset exceeds the length of the UID list").Wrap()
+	}
+
+	end := offset + limit
+	if end >= len(versionInfo.UIDList) {
+		isEnd = true
+		end = len(versionInfo.UIDList)
+	}
+
+	paginatedUIDs := versionInfo.UIDList[offset:end]
+
+	localData, isEnd, err := ds.FetchMissingAndFillLocalV2(ctx, paginatedUIDs, isEnd)
+	if err != nil {
+		return nil, isEnd, err
+	}
+	return localData, isEnd, nil
+}
+
+func (ds *DataFetcher[T]) FetchMissingAndFillLocalV2(ctx context.Context, uids []string, isEnd bool) ([]T, bool, error) {
+	localData, err := ds.FetchFromLocal(ctx, uids)
+	if err != nil {
+		return nil, false, err
+	}
+
+	localUIDSet := datautil.SliceSetAny(localData, ds.Key)
+
+	var missingUIDs []string
+	for _, uid := range uids {
+		if _, found := localUIDSet[uid]; !found {
+			missingUIDs = append(missingUIDs, uid)
+		}
+	}
+
+	if len(missingUIDs) > 0 {
+		serverData, err := ds.fetchFromServer(ctx, missingUIDs)
+		if err != nil {
+			return localData, false, nil
+		}
+
+		if err := ds.batchInsert(ctx, serverData); err != nil {
+			return nil, false, err
+		}
+
+		localData = append(localData, serverData...)
+	}
+
+	return localData, isEnd, nil
+}