Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fetcher: new implementation using unlinked files #1061

Merged
merged 28 commits into from
Oct 12, 2023
Merged

fetcher: new implementation using unlinked files #1061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d413174
zreader: transparent compression helper
hdonnay Sep 11, 2023
031ece5
claircore: add LayerDescription type and FS method
hdonnay Sep 11, 2023
9e3010d
wart: add package to help with API migrations
hdonnay Sep 12, 2023
c73ac54
indexer: add DescriptionRealizer interface
hdonnay Sep 11, 2023
7ca5cf4
test: implement a test fetcher
hdonnay Sep 11, 2023
910dff5
controller: add safety copy
hdonnay Sep 11, 2023
a844586
libindex: implement new unlinked file fetcher
hdonnay Sep 11, 2023
eba5e71
fetch: add ghcr and optional decompression support
hdonnay Aug 22, 2022
fe97f8e
fetch: documentation, switch from `*testing.T` to `testing.TB`
hdonnay Sep 12, 2023
8fbe967
test: documentation fixes
hdonnay Sep 12, 2023
957a02b
test: remove unused `http.RoundTripper` implementation
hdonnay Sep 12, 2023
6f27cfc
test: port various other fetcher abstractions to `claircore.LayerDesc…
hdonnay Sep 12, 2023
46c8758
claircore: update tests for `test` package changes
hdonnay Sep 12, 2023
6375527
test: add matcher for LayerDescriptions and Digests
hdonnay Sep 12, 2023
62508a0
controller: update tests
hdonnay Sep 12, 2023
199573a
whiteout: fix logs in Resolver test
hdonnay Sep 12, 2023
14dca71
whiteout: update tests
hdonnay Sep 12, 2023
26539f2
rpm: update tests
hdonnay Sep 12, 2023
f32073c
rhcc: update tests
hdonnay Sep 12, 2023
9e537ff
rhel: update tests
hdonnay Sep 12, 2023
7baf230
python: update tests
hdonnay Sep 13, 2023
6d1aa81
osrelease: update tests
hdonnay Sep 13, 2023
0c22819
gobin: update tests
hdonnay Sep 13, 2023
60bdacd
dpkg: update tests
hdonnay Sep 13, 2023
62862b0
alpine: update tests
hdonnay Sep 13, 2023
19e1375
claircore: update tests
hdonnay Sep 13, 2023
694fd4b
ruby: update tests
hdonnay Sep 13, 2023
58b6b4a
periodic: update layer handling
hdonnay Sep 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 10 additions & 21 deletions apk/scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,19 @@ package apk

import (
"context"
"net/http"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/quay/zlog"

"github.com/quay/claircore"
"github.com/quay/claircore/test/fetch"
"github.com/quay/claircore/test"
)

func TestScan(t *testing.T) {
hash, err := claircore.ParseDigest("sha256:89d9c30c1d48bac627e5c6cb0d1ed1eec28e7dbdfbcc04712e4c79c0f83faf17")
if err != nil {
t.Fatal(err)
}
t.Parallel()
ctx := zlog.Test(context.Background(), t)
// TODO(hank) Turn into a fixture.
want := []*claircore.Package{
{
Name: "musl",
Expand Down Expand Up @@ -146,22 +144,13 @@ func TestScan(t *testing.T) {
},
}

ctx := zlog.Test(context.Background(), t)
l := &claircore.Layer{
Hash: hash,
}

n, err := fetch.Layer(ctx, t, http.DefaultClient, "docker.io", "library/alpine", hash)
if err != nil {
t.Fatal(err)
}
defer n.Close()

if err := l.SetLocal(n.Name()); err != nil {
t.Error(err)
}
l := test.RealizeLayer(ctx, t, test.LayerRef{
Registry: "docker.io",
Name: "library/alpine",
Digest: "sha256:89d9c30c1d48bac627e5c6cb0d1ed1eec28e7dbdfbcc04712e4c79c0f83faf17",
})
var s Scanner

s := &Scanner{}
got, err := s.Scan(ctx, l)
if err != nil {
t.Fatal(err)
Expand Down
31 changes: 11 additions & 20 deletions dpkg/distroless_scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@ package dpkg

import (
"context"
"net/http"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/quay/claircore"
"github.com/quay/claircore/test/fetch"
"github.com/quay/zlog"

"github.com/quay/claircore"
"github.com/quay/claircore/test"
)

func TestDistrolessLayer(t *testing.T) {
ctx := zlog.Test(context.Background(), t)
want := []*claircore.Package{
{
Name: "base-files",
Expand Down Expand Up @@ -41,25 +42,15 @@ func TestDistrolessLayer(t *testing.T) {
RepositoryHint: "",
},
}
l := test.RealizeLayer(ctx, t, test.LayerRef{
Registry: "gcr.io",
Name: "distroless/static-debian11",
Digest: `sha256:8fdb1fc20e240e9cae976518305db9f9486caa155fd5fc53e7b3a3285fe8a990`,
})
var s DistrolessScanner

t.Parallel()
ctx := zlog.Test(context.Background(), t)
hash := claircore.MustParseDigest(`sha256:8fdb1fc20e240e9cae976518305db9f9486caa155fd5fc53e7b3a3285fe8a990`)
l := claircore.Layer{
Hash: hash,
}
n, err := fetch.Layer(ctx, t, http.DefaultClient, "gcr.io", "distroless/static-debian11", hash)
if err != nil {
t.Fatal(err)
}
defer n.Close()

if err := l.SetLocal(n.Name()); err != nil {
t.Error(err)
}

s := new(DistrolessScanner)
ps, err := s.Scan(ctx, &l)
ps, err := s.Scan(ctx, l)
if err != nil {
t.Error(err)
}
Expand Down
120 changes: 26 additions & 94 deletions dpkg/scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,21 @@ import (
"archive/tar"
"bufio"
"context"
"errors"
"io"
"net/http"
"net/textproto"
"os"
"path/filepath"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/quay/zlog"

"github.com/quay/claircore"
"github.com/quay/claircore/test/fetch"
"github.com/quay/claircore/test/integration"
"github.com/quay/claircore/test"
)

func TestScanner(t *testing.T) {
t.Parallel()
hash, err := claircore.ParseDigest("sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a")
if err != nil {
t.Fatal(err)
}
// TODO(hank) Cook up a manifest format for dpkg ala `test/rpmtest.Manifest`
want := []*claircore.Package{
{
Name: "fdisk",
Expand Down Expand Up @@ -803,21 +796,13 @@ func TestScanner(t *testing.T) {
},
}
ctx := zlog.Test(context.Background(), t)
l := &claircore.Layer{
Hash: hash,
}

n, err := fetch.Layer(ctx, t, http.DefaultClient, "docker.io", "library/ubuntu", hash)
if err != nil {
t.Error(err)
}
defer n.Close()

if err := l.SetLocal(n.Name()); err != nil {
t.Error(err)
}
l := test.RealizeLayer(ctx, t, test.LayerRef{
Registry: "docker.io",
Name: "library/ubuntu",
Digest: "sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a",
})
var s Scanner

s := &Scanner{}
got, err := s.Scan(ctx, l)
if err != nil {
t.Fatal(err)
Expand All @@ -830,25 +815,12 @@ func TestScanner(t *testing.T) {
func TestAbsolutePaths(t *testing.T) {
t.Parallel()
ctx := zlog.Test(context.Background(), t)
hash, err := claircore.ParseDigest("sha256:3c9020349340788076971d5ea638b71e35233fd8e149e269d8eebfa17960c03f")
if err != nil {
t.Fatal(err)
}
l := &claircore.Layer{
Hash: hash,
}

n, err := fetch.Layer(ctx, t, http.DefaultClient, "gcr.io", "vmwarecloudadvocacy/acmeshop-user", hash)
if err != nil {
t.Error(err)
}
defer n.Close()

if err := l.SetLocal(n.Name()); err != nil {
t.Error(err)
}

s := &Scanner{}
l := test.RealizeLayer(ctx, t, test.LayerRef{
Registry: "gcr.io",
Name: "vmwarecloudadvocacy/acmeshop-user",
Digest: "sha256:3c9020349340788076971d5ea638b71e35233fd8e149e269d8eebfa17960c03f",
})
var s Scanner
got, err := s.Scan(ctx, l)
if err != nil {
t.Fatal(err)
Expand All @@ -861,21 +833,21 @@ func TestAbsolutePaths(t *testing.T) {

func TestExtraMetadata(t *testing.T) {
t.Parallel()
layerfile := filepath.Join(integration.PackageCacheDir(t), `extrametadata.layer`)
l := claircore.Layer{
Hash: claircore.MustParseDigest(`sha256:25fd87072f39aaebd1ee24dca825e61d9f5a0f87966c01551d31a4d8d79d37d8`),
URI: "file:///dev/null",
}
mod := test.Modtime(t, "scanner_test.go")
layerfile := test.GenerateFixture(t, `extrametadata.layer`, mod, extraMetadataSetup)
ctx := zlog.Test(context.Background(), t)
var l claircore.Layer
var s Scanner

// Set up the crafted layer
extraMetadataSetup(t, layerfile)
l.SetLocal(layerfile)
if t.Failed() {
return
f, err := os.Open(layerfile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
if err := l.Init(ctx, &test.AnyDescription, f); err != nil {
t.Error(err)
}

s := new(Scanner)
ps, err := s.Scan(ctx, &l)
if err != nil {
t.Error(err)
Expand All @@ -886,47 +858,7 @@ func TestExtraMetadata(t *testing.T) {
}

// ExtraMetadataSetup is a helper to craft a layer that trips PROJQUAY-1308.
func extraMetadataSetup(t *testing.T, layer string) {
t.Helper()

fi, err := os.Stat(layer)
if err != nil {
t.Log(err)
}
switch {
case err == nil:
// If everything looks okay, check if this test has been touched. If so,
// remove the layer and recurse, so that it's re-created.
tf, err := os.Stat(`scanner_test.go`)
if err != nil {
t.Log(err)
}
if !fi.ModTime().After(tf.ModTime()) {
t.Log("recreating layer")
if err := os.Remove(layer); err != nil {
t.Log(err)
}
extraMetadataSetup(t, layer)
}
return
case errors.Is(err, os.ErrNotExist): // OK
os.Mkdir(`testdata`, 0o755)
default:
t.Error(err)
return
}

// If we're here, time to create the layer.
t.Log("creating layer")
f, err := os.Create(layer)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := f.Close(); err != nil {
t.Error(err)
}
}()
func extraMetadataSetup(t testing.TB, f *os.File) {
w := tar.NewWriter(f)
defer func() {
if err := w.Close(); err != nil {
Expand Down
8 changes: 6 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ require (
github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319
github.com/rs/zerolog v1.30.0
github.com/ulikunitz/xz v0.5.11
go.opentelemetry.io/otel v1.18.0
go.opentelemetry.io/otel/trace v1.18.0
golang.org/x/crypto v0.14.0
golang.org/x/sync v0.4.0
golang.org/x/sys v0.13.0
golang.org/x/text v0.13.0
golang.org/x/time v0.3.0
golang.org/x/tools v0.12.0
Expand All @@ -36,6 +39,8 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
Expand All @@ -51,9 +56,8 @@ require (
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
go.opentelemetry.io/otel v1.11.0 // indirect
go.opentelemetry.io/otel/metric v1.18.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/sys v0.13.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
Expand Down
15 changes: 12 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down Expand Up @@ -184,15 +189,19 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk=
go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk=
go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs=
go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ=
go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10=
go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
Expand Down
Loading