From 707f4c85c741334d3a049643babeb629b29483c8 Mon Sep 17 00:00:00 2001 From: tomersein Date: Tue, 20 Aug 2024 08:13:04 +0300 Subject: [PATCH 1/5] add squash all layers resolver Signed-off-by: tomersein --- syft/file/location.go | 4 +- syft/format/syftjson/to_format_model.go | 12 ++ .../container_image_squash_all_layers.go | 174 ++++++++++++++++++ syft/pkg/collection.go | 22 ++- syft/source/scope.go | 5 + syft/source/stereoscopesource/image_source.go | 2 + 6 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 syft/internal/fileresolver/container_image_squash_all_layers.go diff --git a/syft/file/location.go b/syft/file/location.go index 25bc753b97e..004facfd6f8 100644 --- a/syft/file/location.go +++ b/syft/file/location.go @@ -29,7 +29,9 @@ func (l LocationData) Reference() file.Reference { } type LocationMetadata struct { - Annotations map[string]string `json:"annotations,omitempty"` // Arbitrary key-value pairs that can be used to annotate a location + Annotations map[string]string `json:"annotations,omitempty"` // Arbitrary key-value pairs that can be used to annotate a location + IsSquashedAllLayersResolver bool `json:"-"` + IsSquashedLayer bool `json:"-"` } func (m *LocationMetadata) merge(other LocationMetadata) error { diff --git a/syft/format/syftjson/to_format_model.go b/syft/format/syftjson/to_format_model.go index 42ec48f77d2..6c80bb0c24b 100644 --- a/syft/format/syftjson/to_format_model.go +++ b/syft/format/syftjson/to_format_model.go @@ -203,8 +203,20 @@ func toPackageModels(catalog *pkg.Collection, cfg EncoderConfig) []model.Package return artifacts } for _, p := range catalog.Sorted() { + if catalog.IsSquashedAllLayers() { + toDelete := true + for _, l := range p.Locations.ToSlice() { + if l.IsSquashedLayer && l.IsSquashedAllLayersResolver { + toDelete = false + } + } + if toDelete { + continue + } + } artifacts = append(artifacts, toPackageModel(p, cfg)) } + return artifacts } diff --git a/syft/internal/fileresolver/container_image_squash_all_layers.go b/syft/internal/fileresolver/container_image_squash_all_layers.go new file mode 100644 index 00000000000..cf155093bd6 --- /dev/null +++ b/syft/internal/fileresolver/container_image_squash_all_layers.go @@ -0,0 +1,174 @@ +package fileresolver + +import ( + "context" + "io" + + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/syft/file" +) + +var _ file.Resolver = (*ContainerImageSquashAllLayers)(nil) + +// ContainerImageSquashAllLayers implements path and content access for the Squashed all layers source option for container image data sources. +type ContainerImageSquashAllLayers struct { + squashed *ContainerImageSquash + allLayers *ContainerImageAllLayers +} + +// NewFromContainerImageSquashAllLayers returns a new resolver from the perspective of all image layers for the given image. +func NewFromContainerImageSquashAllLayers(img *image.Image) (*ContainerImageSquashAllLayers, error) { + squashed, err := NewFromContainerImageSquash(img) + if err != nil { + return nil, err + } + + allLayers, err := NewFromContainerImageAllLayers(img) + if err != nil { + return nil, err + } + + return &ContainerImageSquashAllLayers{ + squashed: squashed, + allLayers: allLayers, + }, nil +} + +// HasPath indicates if the given path exists in the underlying source. +func (i *ContainerImageSquashAllLayers) HasPath(path string) bool { + return i.squashed.HasPath(path) +} + +// FilesByPath returns all file.References that match the given paths from any layer in the image. +func (i *ContainerImageSquashAllLayers) FilesByPath(paths ...string) ([]file.Location, error) { + squashedLocations, err := i.squashed.FilesByPath(paths...) + if err != nil { + return nil, err + } + + allLayersLocations, err := i.allLayers.FilesByPath(paths...) + if err != nil { + return nil, err + } + + var mergedLocations []file.Location + for _, l := range squashedLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: true, + }, + }) + } + + for _, l := range allLayersLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: false, + }, + }) + } + + return mergedLocations, nil +} + +// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. +// nolint:gocognit +func (i *ContainerImageSquashAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) { + squashedLocations, err := i.squashed.FilesByGlob(patterns...) + if err != nil { + return nil, err + } + + allLayersLocations, err := i.allLayers.FilesByGlob(patterns...) + if err != nil { + return nil, err + } + + var mergedLocations []file.Location + for _, l := range squashedLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: true, + }, + }) + } + + for _, l := range allLayersLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: false, + }, + }) + } + + return mergedLocations, nil +} + +// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. +// This is helpful when attempting to find a file that is in the same layer or lower as another file. +func (i *ContainerImageSquashAllLayers) RelativeFileByPath(location file.Location, path string) *file.Location { + return i.squashed.RelativeFileByPath(location, path) +} + +// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. +// If the path does not exist an error is returned. +func (i *ContainerImageSquashAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + return i.squashed.FileContentsByLocation(location) +} + +func (i *ContainerImageSquashAllLayers) FilesByMIMEType(types ...string) ([]file.Location, error) { + squashedLocations, err := i.squashed.FilesByMIMEType(types...) + if err != nil { + return nil, err + } + + allLayersLocations, err := i.allLayers.FilesByMIMEType(types...) + if err != nil { + return nil, err + } + + var mergedLocations []file.Location + for _, l := range squashedLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: true, + }, + }) + } + + for _, l := range allLayersLocations { + mergedLocations = append(mergedLocations, file.Location{ + LocationData: l.LocationData, + LocationMetadata: file.LocationMetadata{ + Annotations: l.Annotations, + IsSquashedAllLayersResolver: true, + IsSquashedLayer: false, + }, + }) + } + + return mergedLocations, nil +} + +func (i *ContainerImageSquashAllLayers) AllLocations(ctx context.Context) <-chan file.Location { + return i.squashed.AllLocations(ctx) +} + +func (i *ContainerImageSquashAllLayers) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + return i.squashed.FileMetadataByLocation(location) +} diff --git a/syft/pkg/collection.go b/syft/pkg/collection.go index 7ed028b2805..4c71ae3ab6b 100644 --- a/syft/pkg/collection.go +++ b/syft/pkg/collection.go @@ -12,11 +12,12 @@ import ( // Collection represents a collection of Packages. type Collection struct { - byID map[artifact.ID]Package - idsByName map[string]orderedIDSet - idsByType map[Type]orderedIDSet - idsByPath map[string]orderedIDSet // note: this is real path or virtual path - lock sync.RWMutex + byID map[artifact.ID]Package + idsByName map[string]orderedIDSet + idsByType map[Type]orderedIDSet + idsByPath map[string]orderedIDSet // note: this is real path or virtual path + isSquashAllLayer bool + lock sync.RWMutex } // NewCollection returns a new empty Collection @@ -116,6 +117,12 @@ func (c *Collection) add(p Package) { id = p.ID() } + for _, l := range p.Locations.ToSlice() { + if l.IsSquashedAllLayersResolver { + c.isSquashAllLayer = true + } + } + if existing, exists := c.byID[id]; exists { // there is already a package with this fingerprint merge the existing record with the new one if err := existing.merge(p); err != nil { @@ -291,6 +298,11 @@ func (c *Collection) Sorted(types ...Type) (pkgs []Package) { return pkgs } +// IsSquashedAllLayers return if the package collection were used with squashed all layers resolver +func (c *Collection) IsSquashedAllLayers() bool { + return c.isSquashAllLayer +} + type orderedIDSet struct { slice []artifact.ID } diff --git a/syft/source/scope.go b/syft/source/scope.go index b1c9a7b7f64..df355b03862 100644 --- a/syft/source/scope.go +++ b/syft/source/scope.go @@ -12,12 +12,15 @@ const ( SquashedScope Scope = "squashed" // AllLayersScope indicates to catalog content on all layers, regardless if it is visible from the container at runtime. AllLayersScope Scope = "all-layers" + // SquashWithAllLayersScope indicates to catalog content on all layers, but only include content visible from the squashed filesystem representation. + SquashWithAllLayersScope Scope = "squash-with-all-layers" ) // AllScopes is a slice containing all possible scope options var AllScopes = []Scope{ SquashedScope, AllLayersScope, + SquashWithAllLayersScope, } // ParseScope returns a scope as indicated from the given string. @@ -27,6 +30,8 @@ func ParseScope(userStr string) Scope { return SquashedScope case "alllayers", AllLayersScope.String(): return AllLayersScope + case "squash-with-all-layers", strings.ToLower(SquashWithAllLayersScope.String()): + return SquashWithAllLayersScope } return UnknownScope } diff --git a/syft/source/stereoscopesource/image_source.go b/syft/source/stereoscopesource/image_source.go index 71afe3cd8e8..945d1a98edd 100644 --- a/syft/source/stereoscopesource/image_source.go +++ b/syft/source/stereoscopesource/image_source.go @@ -103,6 +103,8 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, res, err = fileresolver.NewFromContainerImageSquash(s.image) case source.AllLayersScope: res, err = fileresolver.NewFromContainerImageAllLayers(s.image) + case source.SquashWithAllLayersScope: + res, err = fileresolver.NewFromContainerImageSquashAllLayers(s.image) default: return nil, fmt.Errorf("bad image scope provided: %+v", scope) } From b22d36a232fce176d1941180cc1926b3068c12b8 Mon Sep 17 00:00:00 2001 From: tomersein Date: Fri, 20 Sep 2024 17:59:44 +0300 Subject: [PATCH 2/5] squashed all layers Signed-off-by: tomersein From f148ddaf4f3b702d0385245212261f07e1a4719e Mon Sep 17 00:00:00 2001 From: tomersein Date: Fri, 20 Sep 2024 18:06:30 +0300 Subject: [PATCH 3/5] squashed all layers Signed-off-by: tomersein From 1d27acce8668c976f7c666a836dfa4053b40a2f9 Mon Sep 17 00:00:00 2001 From: tomersein Date: Fri, 20 Sep 2024 18:07:31 +0300 Subject: [PATCH 4/5] squashed all layers Signed-off-by: tomersein --- syft/internal/fileresolver/container_image_all_layers.go | 3 ++- syft/internal/fileresolver/container_image_squash.go | 3 ++- .../internal/fileresolver/container_image_squash_all_layers.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/syft/internal/fileresolver/container_image_all_layers.go b/syft/internal/fileresolver/container_image_all_layers.go index c1120956b0d..4bad1ef1ba1 100644 --- a/syft/internal/fileresolver/container_image_all_layers.go +++ b/syft/internal/fileresolver/container_image_all_layers.go @@ -120,7 +120,8 @@ func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location, } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -// nolint:gocognit +// +//nolint:gocognit func (r *ContainerImageAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) { uniqueFileIDs := stereoscopeFile.NewFileReferenceSet() uniqueLocations := make([]file.Location, 0) diff --git a/syft/internal/fileresolver/container_image_squash.go b/syft/internal/fileresolver/container_image_squash.go index b3a5ded8914..d3593c9695e 100644 --- a/syft/internal/fileresolver/container_image_squash.go +++ b/syft/internal/fileresolver/container_image_squash.go @@ -79,7 +79,8 @@ func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, er } // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. -// nolint:gocognit +// +//nolint:gocognit func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location, error) { uniqueFileIDs := stereoscopeFile.NewFileReferenceSet() uniqueLocations := make([]file.Location, 0) diff --git a/syft/internal/fileresolver/container_image_squash_all_layers.go b/syft/internal/fileresolver/container_image_squash_all_layers.go index cf155093bd6..b5f5bc20525 100644 --- a/syft/internal/fileresolver/container_image_squash_all_layers.go +++ b/syft/internal/fileresolver/container_image_squash_all_layers.go @@ -78,7 +78,8 @@ func (i *ContainerImageSquashAllLayers) FilesByPath(paths ...string) ([]file.Loc } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -// nolint:gocognit +// +//nolint:gocognit func (i *ContainerImageSquashAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) { squashedLocations, err := i.squashed.FilesByGlob(patterns...) if err != nil { From bc9f627c6ef3ed73e06a7fa48d54566cb730d094 Mon Sep 17 00:00:00 2001 From: tomersein Date: Fri, 20 Sep 2024 18:32:17 +0300 Subject: [PATCH 5/5] squashed all layers Signed-off-by: tomersein --- syft/internal/fileresolver/container_image_squash_all_layers.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/syft/internal/fileresolver/container_image_squash_all_layers.go b/syft/internal/fileresolver/container_image_squash_all_layers.go index b5f5bc20525..e724cb0ae50 100644 --- a/syft/internal/fileresolver/container_image_squash_all_layers.go +++ b/syft/internal/fileresolver/container_image_squash_all_layers.go @@ -78,8 +78,6 @@ func (i *ContainerImageSquashAllLayers) FilesByPath(paths ...string) ([]file.Loc } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -// -//nolint:gocognit func (i *ContainerImageSquashAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) { squashedLocations, err := i.squashed.FilesByGlob(patterns...) if err != nil {