Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make default tarball also oci-flavored #2007

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Make default tarball also oci-flavored #2007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/v1/tarball/progress_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func ExampleWithProgress() {
case update.Error != nil && errors.Is(update.Error, io.EOF):
fmt.Fprintf(os.Stderr, "receive error message: %v\n", err)
fmt.Printf("%d/%d", update.Complete, update.Total)
// Output: 4096/4096
// Output: 8192/8192
return
case update.Error != nil:
fmt.Printf("error writing tarball: %v\n", update.Error)
Expand Down
147 changes: 114 additions & 33 deletions pkg/v1/tarball/write.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,13 @@ import (
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)

const layoutFile = `{
"imageLayoutVersion": "1.0.0"
}`

// WriteToFile writes in the compressed format to a tarball, on disk.
// This is just syntactic sugar wrapping tarball.Write with a new file.
func WriteToFile(p string, ref name.Reference, img v1.Image, opts ...WriteOption) error {
Expand Down Expand Up @@ -99,12 +104,12 @@ func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer, opts ...
}

imageToTags := dedupRefToImage(refToImage)
size, mBytes, err := getSizeAndManifest(imageToTags)
size, mBytes, iBytes, err := getSizeAndManifests(imageToTags)
if err != nil {
return sendUpdateReturn(o, err)
}

return writeImagesToTar(imageToTags, mBytes, size, w, o)
return writeImagesToTar(imageToTags, mBytes, iBytes, size, w, o)
}

// sendUpdateReturn return the passed in error message, also sending on update channel, if it exists
Expand All @@ -126,7 +131,7 @@ func sendProgressWriterReturn(pw *progressWriter, err error) error {
}

// writeImagesToTar writes the images to the tarball
func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w io.Writer, o *writeOptions) (err error) {
func writeImagesToTar(imageToTags map[v1.Image][]string, m, idx []byte, size int64, w io.Writer, o *writeOptions) (err error) {
if w == nil {
return sendUpdateReturn(o, errors.New("must pass valid writer"))
}
Expand All @@ -148,9 +153,40 @@ func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w
tf := tar.NewWriter(tw)
defer tf.Close()

if err := tf.WriteHeader(&tar.Header{
Name: "blobs",
Mode: 0644,
Typeflag: tar.TypeDir,
}); err != nil {
return err
}

if err := tf.WriteHeader(&tar.Header{
Name: "blobs/sha256",
Mode: 0644,
Typeflag: tar.TypeDir,
}); err != nil {
return err
}

seenLayerDigests := make(map[string]struct{})

for img := range imageToTags {
// Write the manifest.
dig, err := img.Digest()
if err != nil {
return sendProgressWriterReturn(pw, err)
}

mFile := fmt.Sprintf("blobs/%s/%s", dig.Algorithm, dig.Hex)
m, err := img.RawManifest()
if err != nil {
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, mFile, bytes.NewReader(m), int64(len(m))); err != nil {
return sendProgressWriterReturn(pw, err)
}

// Write the config.
cfgName, err := img.ConfigName()
if err != nil {
Expand All @@ -160,7 +196,8 @@ func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w
if err != nil {
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil {
configFile := fmt.Sprintf("blobs/%s/%s", cfgName.Algorithm, cfgName.Hex)
if err := writeTarEntry(tf, configFile, bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil {
return sendProgressWriterReturn(pw, err)
}

Expand All @@ -175,21 +212,13 @@ func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w
if err != nil {
return sendProgressWriterReturn(pw, err)
}
// Munge the file name to appease ancient technology.
//
// tar assumes anything with a colon is a remote tape drive:
// https://www.gnu.org/software/tar/manual/html_section/tar_45.html
// Drop the algorithm prefix, e.g. "sha256:"
hex := d.Hex

// gunzip expects certain file extensions:
// https://www.gnu.org/software/gzip/manual/html_node/Overview.html
layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex)
layerFiles[i] = fmt.Sprintf("blobs/%s/%s", d.Algorithm, d.Hex)

if _, ok := seenLayerDigests[hex]; ok {
if _, ok := seenLayerDigests[d.Hex]; ok {
continue
}
seenLayerDigests[hex] = struct{}{}
seenLayerDigests[d.Hex] = struct{}{}

r, err := l.Compressed()
if err != nil {
Expand All @@ -205,9 +234,15 @@ func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w
}
}
}
if err := writeTarEntry(tf, "index.json", bytes.NewReader(idx), int64(len(idx))); err != nil {
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(m), int64(len(m))); err != nil {
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, "oci-layout", strings.NewReader(layoutFile), int64(len(layoutFile))); err != nil {
return sendProgressWriterReturn(pw, err)
}

// be sure to close the tar writer so everything is flushed out before we send our EOF
if err := tf.Close(); err != nil {
Expand All @@ -230,6 +265,8 @@ func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error
return nil, err
}

configFile := fmt.Sprintf("blobs/%s/%s", cfgName.Algorithm, cfgName.Hex)

// Store foreign layer info.
layerSources := make(map[v1.Hash]v1.Descriptor)

Expand All @@ -244,16 +281,8 @@ func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error
if err != nil {
return nil, err
}
// Munge the file name to appease ancient technology.
//
// tar assumes anything with a colon is a remote tape drive:
// https://www.gnu.org/software/tar/manual/html_section/tar_45.html
// Drop the algorithm prefix, e.g. "sha256:"
hex := d.Hex

// gunzip expects certain file extensions:
// https://www.gnu.org/software/gzip/manual/html_node/Overview.html
layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex)
layerFiles[i] = fmt.Sprintf("blobs/%s/%s", d.Algorithm, d.Hex)

// Add to LayerSources if it's a foreign layer.
desc, err := partial.BlobDescriptor(img, d)
Expand All @@ -271,7 +300,7 @@ func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error

// Generate the tar descriptor and write it.
m = append(m, Descriptor{
Config: cfgName.String(),
Config: configFile,
RepoTags: tags,
Layers: layerFiles,
LayerSources: layerSources,
Expand All @@ -286,34 +315,80 @@ func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error
return m, nil
}

// calculateIndex calculates the oci-layout style index
func calculateIndex(imageToTags map[v1.Image][]string) (*v1.IndexManifest, error) {
if len(imageToTags) == 0 {
return nil, errors.New("set of images is empty")
}

idx := v1.IndexManifest{
SchemaVersion: 2,
MediaType: types.OCIImageIndex,
Manifests: make([]v1.Descriptor, 0, len(imageToTags)),
}

// TODO: Tags in here too.
for img := range imageToTags {
desc, err := partial.Descriptor(img)
if err != nil {
return nil, err
}

// Generate the tar descriptor and write it.
idx.Manifests = append(idx.Manifests, *desc)
}

// Sort by size because why not.
sort.Slice(idx.Manifests, func(i, j int) bool {
return idx.Manifests[i].Size < idx.Manifests[j].Size
})

return &idx, nil
}

// CalculateSize calculates the expected complete size of the output tar file
func CalculateSize(refToImage map[name.Reference]v1.Image) (size int64, err error) {
imageToTags := dedupRefToImage(refToImage)
size, _, err = getSizeAndManifest(imageToTags)
size, _, _, err = getSizeAndManifests(imageToTags)
return size, err
}

func getSizeAndManifest(imageToTags map[v1.Image][]string) (int64, []byte, error) {
func getSizeAndManifests(imageToTags map[v1.Image][]string) (int64, []byte, []byte, error) {
m, err := calculateManifest(imageToTags)
if err != nil {
return 0, nil, fmt.Errorf("unable to calculate manifest: %w", err)
return 0, nil, nil, fmt.Errorf("unable to calculate manifest: %w", err)
}
mBytes, err := json.Marshal(m)
if err != nil {
return 0, nil, fmt.Errorf("could not marshall manifest to bytes: %w", err)
return 0, nil, nil, fmt.Errorf("could not marshall manifest to bytes: %w", err)
}

i, err := calculateIndex(imageToTags)
if err != nil {
return 0, nil, nil, fmt.Errorf("calculating index: %w", err)
}
iBytes, err := json.Marshal(i)
if err != nil {
return 0, nil, nil, fmt.Errorf("marshaling index: %w", err)
}

size, err := calculateTarballSize(imageToTags, mBytes)
size, err := calculateTarballSize(imageToTags, mBytes, iBytes)
if err != nil {
return 0, nil, fmt.Errorf("error calculating tarball size: %w", err)
return 0, nil, nil, fmt.Errorf("error calculating tarball size: %w", err)
}
return size, mBytes, nil
return size, mBytes, iBytes, nil
}

// calculateTarballSize calculates the size of the tar file
func calculateTarballSize(imageToTags map[v1.Image][]string, mBytes []byte) (size int64, err error) {
func calculateTarballSize(imageToTags map[v1.Image][]string, mBytes, iBytes []byte) (size int64, err error) {
seenLayerDigests := make(map[string]struct{})
for img, name := range imageToTags {
mSize, err := img.Size()
if err != nil {
return size, fmt.Errorf("unable to get manifest size for img %s: %w", name, err)
}
size += calculateSingleFileInTarSize(mSize)

manifest, err := img.Manifest()
if err != nil {
return size, fmt.Errorf("unable to get manifest for img %s: %w", name, err)
Expand All @@ -328,9 +403,15 @@ func calculateTarballSize(imageToTags map[v1.Image][]string, mBytes []byte) (siz
size += calculateSingleFileInTarSize(l.Size)
}
}

// add the manifest
size += calculateSingleFileInTarSize(int64(len(mBytes)))

// add OCI stuff
size += 1024 // for blobs/sha256 (if sha512 happens oh well this doesn't matter)
size += calculateSingleFileInTarSize(int64(len(layoutFile)))
size += calculateSingleFileInTarSize(int64(len(iBytes)))

// add the two padding blocks that indicate end of a tar file
size += 1024
return size, nil
Expand Down
Loading
Loading