diff --git a/go.mod b/go.mod index 011149b2c0d..a0edcfd3714 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.6.0 github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d - github.com/cs3org/reva/v2 v2.16.2 + github.com/cs3org/reva/v2 v2.16.3 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 2b05e348630..f4fa07c5b66 100644 --- a/go.sum +++ b/go.sum @@ -864,8 +864,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.16.2 h1:ZvbxOsogbPMaXP/XeP/Qt+WluMwZ60wt6/GZhcSFRmY= -github.com/cs3org/reva/v2 v2.16.2/go.mod h1:RvhuweTFqzezjUFU0SIdTXakrEx9vJlMvQ7znPXSP1g= +github.com/cs3org/reva/v2 v2.16.3 h1:FCeKs+4TQevjVd4RymaFrFxaLsuLbCRzUJqlXsts/Ms= +github.com/cs3org/reva/v2 v2.16.3/go.mod h1:RvhuweTFqzezjUFU0SIdTXakrEx9vJlMvQ7znPXSP1g= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go index eeb153777a4..72c2b226403 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/recycle.go @@ -27,6 +27,9 @@ import ( "strings" "time" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" @@ -35,7 +38,6 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storagespace" - "github.com/pkg/errors" ) // Recycle items are stored inside the node folder and start with the uuid of the deleted node. @@ -214,66 +216,111 @@ func readTrashLink(path string) (string, string, string, error) { func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*provider.RecycleItem, error) { log := appctx.GetLogger(ctx) - items := make([]*provider.RecycleItem, 0) - trashRoot := fs.getRecycleRoot(spaceID) - matches, err := filepath.Glob(trashRoot + "/*/*/*/*/*") + + subTrees, err := filepath.Glob(trashRoot + "/*") if err != nil { return nil, err } - for _, itemPath := range matches { - nodePath, nodeID, timeSuffix, err := readTrashLink(itemPath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Msg("error reading trash link, skipping") - continue - } + numWorkers := fs.o.MaxConcurrency + if len(subTrees) < numWorkers { + numWorkers = len(subTrees) + } - md, err := os.Stat(nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not stat trash item, skipping") - continue - } + work := make(chan string, len(subTrees)) + results := make(chan *provider.RecycleItem, len(subTrees)) - attrs, err := fs.lu.MetadataBackend().All(ctx, nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not get extended attributes, skipping") - continue - } + g, ctx := errgroup.WithContext(ctx) - nodeType := fs.lu.TypeFromPath(ctx, nodePath) - if nodeType == provider.ResourceType_RESOURCE_TYPE_INVALID { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("invalid node type, skipping") - continue - } - - item := &provider.RecycleItem{ - Type: nodeType, - Size: uint64(md.Size()), - Key: nodeID, - } - if deletionTime, err := time.Parse(time.RFC3339Nano, timeSuffix); err == nil { - item.DeletionTime = &types.Timestamp{ - Seconds: uint64(deletionTime.Unix()), - // TODO nanos + // Distribute work + g.Go(func() error { + defer close(work) + for _, itemPath := range subTrees { + select { + case work <- itemPath: + case <-ctx.Done(): + return ctx.Err() } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node", nodeID).Str("dtime", timeSuffix).Msg("could not parse time format, ignoring") } + return nil + }) + + // Spawn workers that'll concurrently work the queue + for i := 0; i < numWorkers; i++ { + g.Go(func() error { + for subTree := range work { + matches, err := filepath.Glob(subTree + "/*/*/*/*") + if err != nil { + return err + } + + for _, itemPath := range matches { + nodePath, nodeID, timeSuffix, err := readTrashLink(itemPath) + if err != nil { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Msg("error reading trash link, skipping") + continue + } + + md, err := os.Stat(nodePath) + if err != nil { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not stat trash item, skipping") + continue + } + + attrs, err := fs.lu.MetadataBackend().All(ctx, nodePath) + if err != nil { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not get extended attributes, skipping") + continue + } + + nodeType := fs.lu.TypeFromPath(ctx, nodePath) + if nodeType == provider.ResourceType_RESOURCE_TYPE_INVALID { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("invalid node type, skipping") + continue + } + + item := &provider.RecycleItem{ + Type: nodeType, + Size: uint64(md.Size()), + Key: nodeID, + } + if deletionTime, err := time.Parse(time.RFC3339Nano, timeSuffix); err == nil { + item.DeletionTime = &types.Timestamp{ + Seconds: uint64(deletionTime.Unix()), + // TODO nanos + } + } else { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node", nodeID).Str("dtime", timeSuffix).Msg("could not parse time format, ignoring") + } + + // lookup origin path in extended attributes + if attr, ok := attrs[prefixes.TrashOriginAttr]; ok { + item.Ref = &provider.Reference{Path: string(attr)} + } else { + log.Error().Str("trashRoot", trashRoot).Str("item", itemPath).Str("node", nodeID).Str("dtime", timeSuffix).Msg("could not read origin path") + } + select { + case results <- item: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil + }) + } - // lookup origin path in extended attributes - if attr, ok := attrs[prefixes.TrashOriginAttr]; ok { - item.Ref = &provider.Reference{Path: string(attr)} - } else { - log.Error().Str("trashRoot", trashRoot).Str("item", itemPath).Str("node", nodeID).Str("dtime", timeSuffix).Msg("could not read origin path, skipping") - continue - } - // TODO filter results by permission ... on the original parent? or the trashed node? - // if it were on the original parent it would be possible to see files that were trashed before the current user got access - // so -> check the trash node itself - // hmm listing trash currently lists the current users trash or the 'root' trash. from ocs only the home storage is queried for trash items. - // for now we can only really check if the current user is the owner - items = append(items, item) + // Wait for things to settle down, then close results chan + go func() { + _ = g.Wait() // error is checked later + close(results) + }() + + // Collect results + items := []*provider.RecycleItem{} + for ri := range results { + items = append(items, ri) } return items, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index e4a477f814e..f968e2302d6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,7 +354,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.16.2 +# github.com/cs3org/reva/v2 v2.16.3 ## explicit; go 1.20 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime