From 797156ac89330414f8de1997b793b41711472c12 Mon Sep 17 00:00:00 2001 From: "Kang, Matthew" Date: Thu, 16 Mar 2023 22:13:20 -0700 Subject: [PATCH] added import/export support for OCI compatible image manifest version of cache manifest (opt-in on export, inferred on import) moby/buildkit #2251 Signed-off-by: Kang, Matthew --- README.md | 2 + cache/remotecache/export.go | 60 +++++++++----- cache/remotecache/import.go | 104 ++++++++++++++++++++++--- cache/remotecache/local/local.go | 11 ++- cache/remotecache/registry/registry.go | 11 ++- 5 files changed, 156 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index a08f5cbed1dd..466ef8b27898 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,7 @@ buildctl build ... \ * `min`: only export layers for the resulting image * `max`: export all the layers of all intermediate steps * `ref=`: specify repository reference to store cache, e.g. `docker.io/user/image:tag` +* `image-manifest=`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `false`) * `oci-mediatypes=`: whether to use OCI mediatypes in exported manifests (default: `true`, since BuildKit `v0.8`) * `compression=`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true` * `compression-level=`: choose compression level for gzip, estargz (0-9) and zstd (0-22) @@ -414,6 +415,7 @@ The directory layout conforms to OCI Image Spec v1.0. * `max`: export all the layers of all intermediate steps * `dest=`: destination directory for cache exporter * `tag=`: specify custom tag of image to write to local index (default: `latest`) +* `image-manifest=`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `false`) * `oci-mediatypes=`: whether to use OCI mediatypes in exported manifests (default `true`, since BuildKit `v0.8`) * `compression=`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`. * `compression-level=`: compression level for gzip, estargz (0-9) and zstd (0-22) diff --git a/cache/remotecache/export.go b/cache/remotecache/export.go index b63153d78070..64726361e82d 100644 --- a/cache/remotecache/export.go +++ b/cache/remotecache/export.go @@ -43,18 +43,19 @@ const ( ExporterResponseManifestDesc = "cache.manifest" ) -type contentCacheExporter struct { - solver.CacheExporterTarget - chains *v1.CacheChains - ingester content.Ingester - oci bool - ref string - comp compression.Config +func NewExporter(ingester content.Ingester, ref string, oci bool, imageManifest bool, compressionConfig compression.Config) Exporter { + cc := v1.NewCacheChains() + return &contentCacheExporter{CacheExporterTarget: cc, chains: cc, ingester: ingester, oci: oci, imageManifest: imageManifest, ref: ref, comp: compressionConfig} } -func NewExporter(ingester content.Ingester, ref string, oci bool, compressionConfig compression.Config) Exporter { - cc := v1.NewCacheChains() - return &contentCacheExporter{CacheExporterTarget: cc, chains: cc, ingester: ingester, oci: oci, ref: ref, comp: compressionConfig} +type contentCacheExporter struct { + solver.CacheExporterTarget + chains *v1.CacheChains + ingester content.Ingester + oci bool + imageManifest bool + ref string + comp compression.Config } func (ce *contentCacheExporter) Name() string { @@ -75,20 +76,23 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string } // own type because oci type can't be pushed and docker type doesn't have annotations - type manifestList struct { + type abstractManifest struct { specs.Versioned - MediaType string `json:"mediaType,omitempty"` - + MediaType string `json:"mediaType,omitempty"` + Config *ocispecs.Descriptor `json:"config,omitempty"` // Manifests references platform specific manifests. - Manifests []ocispecs.Descriptor `json:"manifests"` + Manifests []ocispecs.Descriptor `json:"manifests,omitempty"` + Layers []ocispecs.Descriptor `json:"layers,omitempty"` } - var mfst manifestList + var mfst abstractManifest mfst.SchemaVersion = 2 mfst.MediaType = images.MediaTypeDockerSchema2ManifestList - if ce.oci { + if ce.oci && !ce.imageManifest { mfst.MediaType = ocispecs.MediaTypeImageIndex + } else if ce.imageManifest { + mfst.MediaType = ocispecs.MediaTypeImageManifest } for _, l := range config.Layers { @@ -101,10 +105,16 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string return nil, layerDone(errors.Wrap(err, "error writing layer blob")) } layerDone(nil) - mfst.Manifests = append(mfst.Manifests, dgstPair.Descriptor) + if ce.imageManifest { + mfst.Layers = append(mfst.Layers, dgstPair.Descriptor) + } else { + mfst.Manifests = append(mfst.Manifests, dgstPair.Descriptor) + } } - mfst.Manifests = compression.ConvertAllLayerMediaTypes(ctx, ce.oci, mfst.Manifests...) + if !ce.imageManifest { + mfst.Manifests = compression.ConvertAllLayerMediaTypes(ctx, ce.oci, mfst.Manifests...) + } dt, err := json.Marshal(config) if err != nil { @@ -122,7 +132,11 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string } configDone(nil) - mfst.Manifests = append(mfst.Manifests, desc) + if ce.imageManifest { + mfst.Config = &desc + } else { + mfst.Manifests = append(mfst.Manifests, desc) + } dt, err = json.Marshal(mfst) if err != nil { @@ -135,7 +149,12 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string Size: int64(len(dt)), MediaType: mfst.MediaType, } - mfstDone := progress.OneOff(ctx, fmt.Sprintf("writing manifest %s", dgst)) + + mfstLog := fmt.Sprintf("writing cache manifest %s", dgst) + if ce.imageManifest { + mfstLog = fmt.Sprintf("writing cache image manifest %s", dgst) + } + mfstDone := progress.OneOff(ctx, mfstLog) if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil { return nil, mfstDone(errors.Wrap(err, "error writing manifest blob")) } @@ -145,5 +164,6 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string } res[ExporterResponseManifestDesc] = string(descJSON) mfstDone(nil) + return res, nil } diff --git a/cache/remotecache/import.go b/cache/remotecache/import.go index 6e3c6331d9fc..d294c08dbce9 100644 --- a/cache/remotecache/import.go +++ b/cache/remotecache/import.go @@ -3,10 +3,14 @@ package remotecache import ( "context" "encoding/json" + "fmt" "io" "sync" "time" + "github.com/moby/buildkit/util/progress" + "github.com/opencontainers/image-spec/specs-go" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" v1 "github.com/moby/buildkit/cache/remotecache/v1" @@ -21,6 +25,27 @@ import ( "golang.org/x/sync/errgroup" ) +type ManifestType int + +const ( + NotInferred ManifestType = iota + ManifestList + ImageManifest +) + +func (data ManifestType) String() string { + switch data { + case NotInferred: + return "Not Inferred" + case ManifestList: + return "Manifest List" + case ImageManifest: + return "Image Manifest" + default: + return "Not Inferred" + } +} + // ResolveCacheImporterFunc returns importer and descriptor. type ResolveCacheImporterFunc func(ctx context.Context, g session.Group, attrs map[string]string) (Importer, ocispecs.Descriptor, error) @@ -47,24 +72,52 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr return nil, err } - var mfst ocispecs.Index - if err := json.Unmarshal(dt, &mfst); err != nil { + manifestType, err := inferManifestType(ctx, dt) + if err != nil { return nil, err } - allLayers := v1.DescriptorProvider{} + layerDone := progress.OneOff(ctx, fmt.Sprintf("inferred cache manifest type: %s", manifestType)) + layerDone(nil) + allLayers := v1.DescriptorProvider{} var configDesc ocispecs.Descriptor - for _, m := range mfst.Manifests { - if m.MediaType == v1.CacheConfigMediaTypeV0 { - configDesc = m - continue + if manifestType == ManifestList { + var mfst ocispecs.Index + if err := json.Unmarshal(dt, &mfst); err != nil { + return nil, err + } + + for _, m := range mfst.Manifests { + if m.MediaType == v1.CacheConfigMediaTypeV0 { + configDesc = m + continue + } + allLayers[m.Digest] = v1.DescriptorProviderPair{ + Descriptor: m, + Provider: ci.provider, + } } - allLayers[m.Digest] = v1.DescriptorProviderPair{ - Descriptor: m, - Provider: ci.provider, + } else if manifestType == ImageManifest { + var mfst ocispecs.Manifest + if err := json.Unmarshal(dt, &mfst); err != nil { + return nil, err } + + for _, m := range mfst.Layers { + if m.MediaType == v1.CacheConfigMediaTypeV0 { + configDesc = m + continue + } + allLayers[m.Digest] = v1.DescriptorProviderPair{ + Descriptor: m, + Provider: ci.provider, + } + } + } else { + err = errors.Wrapf(err, "Unsupported or uninferrable manifest type") + return nil, err } if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok { @@ -97,6 +150,37 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr return solver.NewCacheManager(ctx, id, keysStorage, resultStorage), nil } +// extends support for "new"-style image-manifest style remote cache manifests and determining downstream +// handling based on inference of document structure (is this a new or old cache manifest type?) +func inferManifestType(ctx context.Context, dt []byte) (ManifestType, error) { + // this is a loose schema superset of both OCI Index and Manifest in order to + // be able to poke at the structure of the imported cache manifest + type OpenManifest struct { + specs.Versioned + + MediaType string `json:"mediaType,omitempty"` + Config map[string]interface{} `json:"config,omitempty"` + // Manifests references platform specific manifests. + Manifests []map[string]interface{} `json:"manifests,omitempty"` + Layers []map[string]interface{} `json:"layers,omitempty"` + } + + var openManifest OpenManifest + if err := json.Unmarshal(dt, &openManifest); err != nil { + return NotInferred, err + } + + if len(openManifest.Manifests) == 0 && len(openManifest.Layers) > 0 { + return ImageManifest, nil + } + + if len(openManifest.Layers) == 0 && len(openManifest.Manifests) > 0 { + return ManifestList, nil + } + + return NotInferred, nil +} + func readBlob(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor) ([]byte, error) { maxBlobSize := int64(1 << 20) if desc.Size > maxBlobSize { diff --git a/cache/remotecache/local/local.go b/cache/remotecache/local/local.go index 9bff2cc794a2..818f9b441ee4 100644 --- a/cache/remotecache/local/local.go +++ b/cache/remotecache/local/local.go @@ -19,6 +19,7 @@ const ( attrDigest = "digest" attrSrc = "src" attrDest = "dest" + attrImageManifest = "image-manifest" attrOCIMediatypes = "oci-mediatypes" contentStoreIDPrefix = "local:" ) @@ -50,12 +51,20 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor } ociMediatypes = b } + imageManifest := false + if v, ok := attrs[attrImageManifest]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", attrImageManifest) + } + imageManifest = b + } csID := contentStoreIDPrefix + store cs, err := getContentStore(ctx, sm, g, csID) if err != nil { return nil, err } - return &exporter{remotecache.NewExporter(cs, "", ociMediatypes, compressionConfig)}, nil + return &exporter{remotecache.NewExporter(cs, "", ociMediatypes, imageManifest, compressionConfig)}, nil } } diff --git a/cache/remotecache/registry/registry.go b/cache/remotecache/registry/registry.go index e82638ebf699..007da9885509 100644 --- a/cache/remotecache/registry/registry.go +++ b/cache/remotecache/registry/registry.go @@ -36,6 +36,7 @@ func canonicalizeRef(rawRef string) (reference.Named, error) { const ( attrRef = "ref" + attrImageManifest = "image-manifest" attrOCIMediatypes = "oci-mediatypes" attrInsecure = "registry.insecure" ) @@ -67,6 +68,14 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r } ociMediatypes = b } + imageManifest := false + if v, ok := attrs[attrImageManifest]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", attrImageManifest) + } + imageManifest = b + } insecure := false if v, ok := attrs[attrInsecure]; ok { b, err := strconv.ParseBool(v) @@ -82,7 +91,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r if err != nil { return nil, err } - return &exporter{remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, compressionConfig)}, nil + return &exporter{remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, imageManifest, compressionConfig)}, nil } }