Support pagination for DIDs (#518)

* Support pagination for DIDs

* Spec

* Name

* Unused

* Just use ints

* Spec

* review

* Added integration test for List with paging

doc/swagger.yaml

@@ -1514,6 +1514,10 @@ definitions:
  items:
  $ref: '#/definitions/did.Document'
  type: array
+ nextPageToken:
+ description: Pagination token to retrieve the next page of results. If the
+ value is "", it means no further results for the request.
+ type: string
  type: object
  pkg_server_router.ListDefinitionsResponse:
  properties:
@@ -2284,6 +2288,16 @@ paths:
  in: query
  name: deleted
  type: boolean
+ - description: Hint to the server of the maximum elements to return. More may
+ be returned. When not set, the server will return all elements.
+ in: query
+ name: pageSize
+ type: number
+ - description: Used to indicate to the server to return a specific page of the
+ list results. Must match a previous requests' `nextPageToken`.
+ in: query
+ name: pageToken
+ type: string
  produces:
  - application/json
  responses:

integration/common.go

@@ -7,6 +7,7 @@ import (
  "fmt"
  "io"
  "net/http"
+ "net/url"
  "text/template"
  "time"
 
@@ -86,6 +87,18 @@ func CreateDIDION() (string, error) {
  return output, nil
 }
 
+func ListWebDIDs() (string, error) {
+ urlValues := url.Values{
+ "pageSize": []string{"10"},
+ }
+ output, err := get(endpoint + version + "dids/web?" + urlValues.Encode())
+ if err != nil {
+ return "", errors.Wrapf(err, "list web did")
+ }
+
+ return output, nil
+}
+
 func ResolveDID(did string) (string, error) {
  logrus.Println("\n\nResolve a did")
  output, err := get(endpoint + version + "dids/resolver/" + did)

integration/didweb_integration_test.go

@@ -31,6 +31,21 @@ func TestCreateIssuerDIDWebIntegration(t *testing.T) {
  SetValue(didWebContext, "issuerKID", issuerKID)
 }
 
+func TestListDIDWebIntegration(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping integration test")
+ }
+
+ listWebDIDsOutput, err := ListWebDIDs()
+ assert.NoError(t, err)
+
+ issuerDID, err := GetValue(didWebContext, "issuerDID")
+ assert.NoError(t, err)
+ assert.NotEmpty(t, issuerDID)
+
+ assert.Contains(t, listWebDIDsOutput, issuerDID)
+}
+
 func TestCreateAliceDIDKeyForDIDWebIntegration(t *testing.T) {
  if testing.Short() {
  t.Skip("skipping integration test")

pkg/server/router/did.go

@@ -1,8 +1,11 @@
 package router
 
 import (
+ "encoding/base64"
  "fmt"
  "net/http"
+ "net/url"
+ "reflect"
  "strconv"
 
  "github.com/TBD54566975/ssi-sdk/crypto"
@@ -11,6 +14,7 @@ import (
  "github.com/gin-gonic/gin"
  "github.com/goccy/go-json"
  "github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
 
  "github.com/tbd54566975/ssi-service/internal/util"
  "github.com/tbd54566975/ssi-service/pkg/server/framework"
@@ -19,9 +23,11 @@ import (
 )
 
 const (
- MethodParam = "method"
- IDParam = "id"
- DeletedParam = "deleted"
+ MethodParam = "method"
+ IDParam = "id"
+ DeletedParam = "deleted"
+ PageSizeParam = "pageSize"
+ PageTokenParam = "pageToken"
 )
 
 // DIDRouter represents the dependencies required to instantiate a DID-HTTP service
@@ -224,6 +230,9 @@ func (dr DIDRouter) GetDIDByMethod(c *gin.Context) {
 
 type ListDIDsByMethodResponse struct {
  DIDs []didsdk.Document `json:"dids,omitempty"`
+
+ // Pagination token to retrieve the next page of results. If the value is "", it means no further results for the request.
+ NextPageToken string `json:"nextPageToken"`
 }
 
 type GetDIDsRequest struct {
@@ -232,27 +241,34 @@ type GetDIDsRequest struct {
  Filter string `json:"filter,omitempty"`
 }
 
+type PageToken struct {
+ EncodedQuery string
+ NextPageToken string
+}
+
 // ListDIDsByMethod godoc
 //
 // @Summary List DIDs
 // @Description List DIDs by method. Checks for an optional "deleted=true" query parameter, which exclusively returns DIDs that have been "Soft Deleted".
 // @Tags DecentralizedIdentityAPI
 // @Accept json
 // @Produce json
-// @Param deleted query boolean false "When true, returns soft-deleted DIDs. Otherwise, returns DIDs that have not been soft-deleted. Default is false."
-// @Success 200 {object} ListDIDsByMethodResponse
-// @Failure 400 {string} string "Bad request"
-// @Failure 500 {string} string "Internal server error"
+// @Param deleted query boolean false "When true, returns soft-deleted DIDs. Otherwise, returns DIDs that have not been soft-deleted. Default is false."
+// @Param pageSize query number false "Hint to the server of the maximum elements to return. More may be returned. When not set, the server will return all elements."
+// @Param pageToken query string false "Used to indicate to the server to return a specific page of the list results. Must match a previous requests' `nextPageToken`."
+// @Success 200 {object} ListDIDsByMethodResponse
+// @Failure 400 {string} string "Bad request"
+// @Failure 500 {string} string "Internal server error"
 // @Router /v1/dids/{method} [get]
 func (dr DIDRouter) ListDIDsByMethod(c *gin.Context) {
  method := framework.GetParam(c, MethodParam)
- deleted := framework.GetQueryValue(c, DeletedParam)
  if method == nil {
  errMsg := "list DIDs by method request missing method parameter"
  framework.LoggingRespondErrMsg(c, errMsg, http.StatusBadRequest)
  return
  }
  getIsDeleted := false
+ deleted := framework.GetQueryValue(c, DeletedParam)
  if deleted != nil {
  checkDeleted, err := strconv.ParseBool(*deleted)
  getIsDeleted = checkDeleted
@@ -263,21 +279,83 @@ func (dr DIDRouter) ListDIDsByMethod(c *gin.Context) {
  return
  }
  }
-
  // TODO(gabe) check if the method is supported, to tell whether this is a bad req or internal error
  // TODO(gabe) differentiate between internal errors and not found DIDs
  getDIDsRequest := did.ListDIDsRequest{Method: didsdk.Method(*method), Deleted: getIsDeleted}
- gotDIDs, err := dr.service.ListDIDsByMethod(c, getDIDsRequest)
+
+ pageSizeStr := framework.GetParam(c, PageSizeParam)
+
+ if pageSizeStr != nil {
+ pageSize, err := strconv.Atoi(*pageSizeStr)
+ if err != nil {
+ errMsg := fmt.Sprintf("list DIDs by method request encountered a problem with the %q query param", PageSizeParam)
+ framework.LoggingRespondErrMsg(c, errMsg, http.StatusBadRequest)
+ return
+ }
+ getDIDsRequest.PageSize = &pageSize
+ }
+
+ queryPageToken := framework.GetParam(c, PageTokenParam)
+ if queryPageToken != nil {
+ errMsg := "token value cannot be decoded"
+ tokenData, err := base64.RawURLEncoding.DecodeString(*queryPageToken)
+ if err != nil {
+ framework.LoggingRespondErrMsg(c, errMsg, http.StatusBadRequest)
+ return
+ }
+ var pageToken PageToken
+ if err := json.Unmarshal(tokenData, &pageToken); err != nil {
+ framework.LoggingRespondErrMsg(c, errMsg, http.StatusBadRequest)
+ return
+ }
+ pageTokenValues, err := url.ParseQuery(pageToken.EncodedQuery)
+ if err != nil {
+ framework.LoggingRespondErrMsg(c, errMsg, http.StatusBadRequest)
+ return
+ }
+
+ query := pageTokenQuery(c)
+ if !reflect.DeepEqual(pageTokenValues, query) {
+ logrus.Warnf("expected query from token to be equal to query from request. token: %v\nrequest%v", pageTokenValues, query)
+ framework.LoggingRespondErrMsg(c, "page token must be for the same query", http.StatusBadRequest)
+ return
+ }
+ getDIDsRequest.PageToken = &pageToken.NextPageToken
+ }
+
+ listResp, err := dr.service.ListDIDsByMethod(c, getDIDsRequest)
  if err != nil {
  errMsg := fmt.Sprintf("could not get DIDs for method: %s", *method)
  framework.LoggingRespondErrWithMsg(c, err, errMsg, http.StatusInternalServerError)
  return
  }
 
- resp := ListDIDsByMethodResponse{DIDs: gotDIDs.DIDs}
+ resp := ListDIDsByMethodResponse{
+ DIDs: listResp.DIDs,
+ }
+ if listResp.NextPageToken != "" {
+ tokenQuery := pageTokenQuery(c)
+ pageToken := PageToken{
+ EncodedQuery: tokenQuery.Encode(),
+ NextPageToken: listResp.NextPageToken,
+ }
+ nextPageTokenData, err := json.Marshal(pageToken)
+ if err != nil {
+ framework.LoggingRespondErrWithMsg(c, err, "marshalling page token", http.StatusInternalServerError)
+ return
+ }
+ resp.NextPageToken = base64.RawURLEncoding.EncodeToString(nextPageTokenData)
+ }
  framework.Respond(c, resp, http.StatusOK)
 }
 
+func pageTokenQuery(c *gin.Context) url.Values {
+ query := c.Request.URL.Query()
+ delete(query, PageTokenParam)
+ delete(query, PageSizeParam)
+ return query
+}
+
 type ResolveDIDResponse struct {
  ResolutionMetadata *resolution.Metadata `json:"didResolutionMetadata,omitempty"`
  DIDDocument *didsdk.Document `json:"didDocument"`

pkg/server/router/did_test.go

@@ -28,8 +28,40 @@ func TestDIDRouter(t *testing.T) {
  assert.Contains(tt, err.Error(), "could not create DID router with service type: test")
  })
 
- t.Run("DID Service Test", func(tt *testing.T) {
+ t.Run("List DIDs supports paging", func(tt *testing.T) {
+ db := setupTestDB(tt)
+ assert.NotEmpty(tt, db)
+ keyStoreService := testKeyStoreService(tt, db)
+ methods := []string{didsdk.KeyMethod.String()}
+ serviceConfig := config.DIDServiceConfig{Methods: methods, LocalResolutionMethods: methods}
+ didService, err := did.NewDIDService(serviceConfig, db, keyStoreService)
+ assert.NoError(tt, err)
+ assert.NotEmpty(tt, didService)
+ createDID(tt, didService)
+ createDID(tt, didService)
+
+ one := 1
+ listDIDsResponse1, err := didService.ListDIDsByMethod(context.Background(),
+ did.ListDIDsRequest{
+ Method: didsdk.KeyMethod,
+ PageSize: &one,
+ })
+ assert.NoError(tt, err)
+ assert.Len(tt, listDIDsResponse1.DIDs, 1)
+ assert.NotEmpty(tt, listDIDsResponse1.NextPageToken)
+
+ listDIDsResponse2, err := didService.ListDIDsByMethod(context.Background(),
+ did.ListDIDsRequest{
+ Method: didsdk.KeyMethod,
+ PageSize: &one,
+ PageToken: &listDIDsResponse1.NextPageToken,
+ })
+ assert.NoError(tt, err)
+ assert.Len(tt, listDIDsResponse2.DIDs, 1)
+ assert.Empty(tt, listDIDsResponse2.NextPageToken)
+ })
 
+ t.Run("DID Service Test", func(tt *testing.T) {
  db := setupTestDB(tt)
  assert.NotEmpty(tt, db)
 
@@ -117,7 +149,6 @@ func TestDIDRouter(t *testing.T) {
  })
 
  t.Run("DID Web Service Test", func(tt *testing.T) {
-
  db := setupTestDB(tt)
  assert.NotEmpty(tt, db)
 
@@ -202,3 +233,9 @@ func TestDIDRouter(t *testing.T) {
  })
 
 }
+
+func createDID(tt *testing.T, didService *did.Service) {
+ createDIDResponse, err := didService.CreateDIDByMethod(context.Background(), did.CreateDIDRequest{Method: didsdk.KeyMethod, KeyType: crypto.Ed25519})
+ assert.NoError(tt, err)
+ assert.NotEmpty(tt, createDIDResponse)
+}