diff --git a/container.go b/container.go index d8e2c7c2f3..d3457a0833 100644 --- a/container.go +++ b/container.go @@ -125,7 +125,7 @@ type ContainerRequest struct { User string // for specifying uid:gid SkipReaper bool // Deprecated: The reaper is globally controlled by the .testcontainers.properties file or the TESTCONTAINERS_RYUK_DISABLED environment variable ReaperImage string // Deprecated: use WithImageName ContainerOption instead. Alternative reaper image - ReaperOptions []ContainerOption // options for the reaper + ReaperOptions []ContainerOption // Deprecated: the reaper is configured at the properties level, for an entire test session AutoRemove bool // Deprecated: Use HostConfigModifier instead. If set to true, the container will be removed from the host when stopped AlwaysPullImage bool // Always pull image ImagePlatform string // ImagePlatform describes the platform which the image runs on. @@ -145,9 +145,11 @@ type containerOptions struct { RegistryCredentials string // Deprecated: Testcontainers will detect registry credentials automatically } +// Deprecated: it will be removed in the next major release // functional option for setting the reaper image type ContainerOption func(*containerOptions) +// Deprecated: it will be removed in the next major release // WithImageName sets the reaper image name func WithImageName(imageName string) ContainerOption { return func(o *containerOptions) { @@ -155,7 +157,7 @@ func WithImageName(imageName string) ContainerOption { } } -// Deprecated: Testcontainers will detect registry credentials automatically +// Deprecated: Testcontainers will detect registry credentials automatically, and it will be removed in the next major release // WithRegistryCredentials sets the reaper registry credentials func WithRegistryCredentials(registryCredentials string) ContainerOption { return func(o *containerOptions) { diff --git a/container_test.go b/container_test.go index 0de6f33a19..423f6fcbb6 100644 --- a/container_test.go +++ b/container_test.go @@ -306,11 +306,13 @@ func Test_BuildImageWithContexts(t *testing.T) { func Test_GetLogsFromFailedContainer(t *testing.T) { ctx := context.Background() + // directDockerHubReference { req := ContainerRequest{ Image: "docker.io/alpine", Cmd: []string{"echo", "-n", "I was not expecting this"}, WaitingFor: wait.ForLog("I was expecting this").WithStartupTimeout(5 * time.Second), } + // } c, err := GenericContainer(ctx, GenericContainerRequest{ ContainerRequest: req, @@ -340,6 +342,7 @@ func Test_GetLogsFromFailedContainer(t *testing.T) { } } +// dockerImageSubstitutor { type dockerImageSubstitutor struct{} func (s dockerImageSubstitutor) Description() string { @@ -350,6 +353,8 @@ func (s dockerImageSubstitutor) Substitute(image string) (string, error) { return "docker.io/" + image, nil } +// } + // noopImageSubstitutor { type NoopImageSubstitutor struct{} @@ -506,3 +511,35 @@ func TestParseDockerIgnore(t *testing.T) { assert.Equal(t, testCase.expectedExcluded, excluded) } } + +func ExampleGenericContainer_withSubstitutors() { + ctx := context.Background() + + // applyImageSubstitutors { + container, err := GenericContainer(ctx, GenericContainerRequest{ + ContainerRequest: ContainerRequest{ + Image: "alpine:latest", + ImageSubstitutors: []ImageSubstitutor{dockerImageSubstitutor{}}, + }, + Started: true, + }) + // } + if err != nil { + panic(err) + } + + defer func() { + err := container.Terminate(ctx) + if err != nil { + panic(err) + } + }() + + // enforce the concrete type, as GenericContainer returns an interface, + // which will be changed in future implementations of the library + dockerContainer := container.(*DockerContainer) + + fmt.Println(dockerContainer.Image) + + // Output: docker.io/alpine:latest +} diff --git a/docker.go b/docker.go index 2d657abad8..c1d84d70de 100644 --- a/docker.go +++ b/docker.go @@ -31,6 +31,7 @@ import ( specs "github.com/opencontainers/image-spec/specs-go/v1" tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/internal/config" "github.com/testcontainers/testcontainers-go/internal/testcontainersdocker" "github.com/testcontainers/testcontainers-go/internal/testcontainerssession" "github.com/testcontainers/testcontainers-go/wait" @@ -882,20 +883,13 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque req.Labels = make(map[string]string) } - reaperOpts := containerOptions{ - ImageName: req.ReaperImage, - } - for _, opt := range req.ReaperOptions { - opt(&reaperOpts) - } - tcConfig := p.Config().Config var termSignal chan bool // the reaper does not need to start a reaper for itself - isReaperContainer := strings.EqualFold(imageName, reaperImage(reaperOpts.ImageName)) + isReaperContainer := strings.HasSuffix(imageName, config.ReaperDefaultImage) if !tcConfig.RyukDisabled && !isReaperContainer { - r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), testcontainerssession.SessionID(), p, req.ReaperOptions...) + r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), testcontainerssession.SessionID(), p) if err != nil { return nil, fmt.Errorf("%w: creating reaper failed", err) } @@ -916,14 +910,19 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque return nil, err } + // always append the hub substitutor after the user-defined ones + req.ImageSubstitutors = append(req.ImageSubstitutors, newPrependHubRegistry()) + for _, is := range req.ImageSubstitutors { modifiedTag, err := is.Substitute(imageName) if err != nil { return nil, fmt.Errorf("failed to substitute image %s with %s: %w", imageName, is.Description(), err) } - p.Logger.Printf("✍🏼 Replacing image with %s. From: %s to %s\n", is.Description(), imageName, modifiedTag) - imageName = modifiedTag + if modifiedTag != imageName { + p.Logger.Printf("✍🏼 Replacing image with %s. From: %s to %s\n", is.Description(), imageName, modifiedTag) + imageName = modifiedTag + } } var platform *specs.Platform @@ -1146,7 +1145,7 @@ func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req Contain var termSignal chan bool if !tcConfig.RyukDisabled { - r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p, req.ReaperOptions...) + r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p) if err != nil { return nil, fmt.Errorf("%w: creating reaper failed", err) } @@ -1314,7 +1313,7 @@ func (p *DockerProvider) CreateNetwork(ctx context.Context, req NetworkRequest) var termSignal chan bool if !tcConfig.RyukDisabled { - r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p, req.ReaperOptions...) + r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p) if err != nil { return nil, fmt.Errorf("%w: creating network reaper failed", err) } diff --git a/docs/features/common_functional_options.md b/docs/features/common_functional_options.md index e3ebbde1e7..b45cd54d39 100644 --- a/docs/features/common_functional_options.md +++ b/docs/features/common_functional_options.md @@ -2,7 +2,16 @@ - Since testcontainers-go :material-tag: v0.26.0 -{% include "./image_name_substitution.md" %} +In more locked down / secured environments, it can be problematic to pull images from Docker Hub and run them without additional precautions. + +An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry. + +_Testcontainers for Go_ exposes an interface to perform this operations: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations: + + +[Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor +[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor + Using the `WithImageSubstitutors` options, you could define your own substitutions to the container images. E.g. adding a prefix to the images so that they can be pulled from a Docker registry other than Docker Hub. This is the usual mechanism for using Docker image proxies, caches, etc. diff --git a/docs/features/configuration.md b/docs/features/configuration.md index c6d921a85e..ede9dbcae5 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -40,11 +40,19 @@ docker.tls.verify=1 # Equivalent to the DOCKER_TLS_VERIF docker.cert.path=/some/path # Equivalent to the DOCKER_CERT_PATH environment variable ``` -### Disabling Ryuk -Ryuk must be started as a privileged container. -If your environment already implements automatic cleanup of containers after the execution, +## Customizing images + +Please read more about customizing images in the [Image name substitution](image_name_substitution.md) section. + +## Customizing Ryuk, the resource reaper + +1. Ryuk must be started as a privileged container. For that, you can set the `TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED` **environment variable**, or the `ryuk.container.privileged` **property** to `true`. +1. If your environment already implements automatic cleanup of containers after the execution, but does not allow starting privileged containers, you can turn off the Ryuk container by setting `TESTCONTAINERS_RYUK_DISABLED` **environment variable** to `true`. +1. You can specify the connection timeout for Ryuk by setting the `ryuk.connection.timeout` **property**. The default value is 1 minute. +1. You can specify the reconnection timeout for Ryuk by setting the `ryuk.reconnection.timeout` **property**. The default value is 10 seconds. + !!!info For more information about Ryuk, see [Garbage Collector](garbage_collector.md). diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index fe8729e9c7..59693d3fcb 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -1,10 +1,95 @@ -In more locked down / secured environments, it can be problematic to pull images from Docker Hub and run them without additional precautions. +# Image name substitution -An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry. +_Testcontainers for Go_ supports automatic substitution of Docker image names. -_Testcontainers for Go_ exposes an interface to perform this operations: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations: +This allows the replacement of an image name specified in test code with an alternative name - for example, to replace the +name of a Docker Hub image dependency with an alternative hosted on a private image registry. + +This is advisable to avoid Docker Hub rate limiting, and some companies will prefer this for policy reasons. + +!!!info + As of November 2020 Docker Hub pulls are rate limited. As Testcontainers uses Docker Hub for standard images, some users may hit these rate limits and should mitigate accordingly. Suggested mitigations are noted in [this issue in Testcontainers for Java](https://github.com/testcontainers/testcontainers-java/issues/3099) at present. + +This page describes two approaches for image name substitution: + +* [Automatically modifying Docker Hub image names](#automatically-modifying-docker-hub-image-names), prefixing them with a private registry URL. +* [Using an Image Name Substitutor](#developing-a-custom-function-for-transforming-image-names-on-the-fly), developing a custom function for transforming image names on the fly. + +!!!warning + It is assumed that you have already set up a private registry hosting [all the Docker images your build requires](../supported_docker_environment/image_registry_rate_limiting.md#which-images-are-used-by-testcontainers). + +## Automatically modifying Docker Hub image names + +_Testcontainers for Go_ can be configured to modify Docker Hub image names on the fly to apply a prefix string. + +Consider this if: + +* Developers and CI machines need to use different image names. For example, developers are able to pull images from Docker Hub, but CI machines need to pull from a private registry. +* Your private registry has copies of images from Docker Hub where the names are predictable, and just adding a prefix is enough. + For example, `registry.mycompany.com/mirror/mysql:8.0.24` can be derived from the original Docker Hub image name (`mysql:8.0.24`) with a consistent prefix string: `registry.mycompany.com/mirror` + +In this case, image name references in code are **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../../container_test.go) inside_block:directDockerHubReference + + +You can then configure _Testcontainers for Go_ to apply a given prefix (e.g. `registry.mycompany.com/mirror`) to every image that it tries to pull from Docker Hub. Important to notice that **the prefix should not include a trailing slash**. This can be done in one of two ways: + +* Setting the `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.mycompany.com/mirror` environment variable. +* Via config file, setting `hub.image.name.prefix` in the `~/.testcontainers.properties` file in your user home directory. + +_Testcontainers for Go_ will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](#images-used-by-testcontainers) exist in your registry. + +_Testcontainers for Go_ will not apply the prefix to: + +* non-Hub image names (e.g. where another registry is set) +* Docker Hub image names where the hub registry is explicitly part of the name (i.e. anything with a `docker.io` or `registry.hub.docker.com` host part) + +## Developing a custom function for transforming image names on the fly + +Consider this if: + +* You have complex rules about which private registry images should be used as substitutes, e.g.: + * non-deterministic mapping of names meaning that a [name prefix](#automatically-modifying-docker-hub-image-names) cannot be used, or + * rules depending upon developer identity or location, or +* you wish to add audit logging of images used in the build, or +* you wish to prevent accidental usage of images that are not on an approved list. + +In this case, image name references in code are **unchanged**. i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../../container_test.go) inside_block:directDockerHubReference + + +You can implement a custom image name substitutor by: + +* implementing the `ImageNameSubstitutor` interface, exposed by the `testcontainers` package. +* configuring _Testcontainers for Go_ to use your custom implementation, defined at the `ContainerRequest` level. + +The following is an example image substitutor implementation prepending the `docker.io/` prefix, used in the tests: [Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor -[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor +[Docker prefix Image Substitutor](../../container_test.go) inside_block:dockerImageSubstitutor +[Applying the substitutor](../../container_test.go) inside_block:applyImageSubstitutors + +## Images used by Testcontainers + +As of the current version of Testcontainers ({{latest_version}}): + +* every image directly used by your tests +* images pulled by Testcontainers itself to support functionality: + * [`testcontainers/ryuk`](https://hub.docker.com/r/testcontainers/ryuk) - performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](./configuration.md#customizing-ryuk-the-resource-reaper)). + * [`alpine`](https://hub.docker.com/r/_/alpine). + * [`Docker in Docker`](https://hub.docker.com/_/docker). + * [`nginx`](https://hub.docker.com/r/_/nginx). + * [`delayed nginx`](https://hub.docker.com/r/menedev/delayed-nginx). + * [`localstack`](https://hub.docker.com/r/localstack/localstack). + * [`mysql`](https://hub.docker.com/r/_/mysql). + * [`postgres`](https://hub.docker.com/r/_/postgres). + * [`postgis`](https://hub.docker.com/r/postgis/postgis). + * [`redis`](https://hub.docker.com/r/_/redis). + * [`registry`](https://hub.docker.com/r/_/registry). diff --git a/docs/system_requirements/ci/bitbucket_pipelines.md b/docs/system_requirements/ci/bitbucket_pipelines.md index 9e0e2f50a0..0c83ca42ad 100644 --- a/docs/system_requirements/ci/bitbucket_pipelines.md +++ b/docs/system_requirements/ci/bitbucket_pipelines.md @@ -2,7 +2,7 @@ To enable access to Docker in Bitbucket Pipelines, you need to add `docker` as a service on the step. -Furthermore, Ryuk needs to be turned off since Bitbucket Pipelines does not allow starting privileged containers (see [Disabling Ryuk](../../features/configuration.md#disabling-ryuk)). This can either be done by setting a repository variable in Bitbucket's project settings or by explicitly exporting the variable on a step. +Furthermore, Ryuk needs to be turned off since Bitbucket Pipelines does not allow starting privileged containers (see [Disabling Ryuk](../../features/configuration.md#customizing-ryuk-the-resource-reaper)). This can either be done by setting a repository variable in Bitbucket's project settings or by explicitly exporting the variable on a step. In some cases the memory available to Docker needs to be increased. diff --git a/image_substitutors_test.go b/image_substitutors_test.go new file mode 100644 index 0000000000..e73c4718f8 --- /dev/null +++ b/image_substitutors_test.go @@ -0,0 +1,112 @@ +package testcontainers + +import ( + "testing" + + "github.com/testcontainers/testcontainers-go/internal/config" +) + +func TestPrependHubRegistrySubstitutor(t *testing.T) { + resetConfigForTests := func() { + config.Reset() + } + + t.Run("should prepend the hub registry to images from Docker Hub", func(t *testing.T) { + t.Run("plain image", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "my-registry/foo:latest" { + t.Errorf("expected my-registry/foo, got %s", img) + } + }) + t.Run("image with user", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("user/foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "my-registry/user/foo:latest" { + t.Errorf("expected my-registry/foo, got %s", img) + } + }) + + t.Run("image with organization and user", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("org/user/foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "my-registry/org/user/foo:latest" { + t.Errorf("expected my-registry/org/foo:latest, got %s", img) + } + }) + }) + + t.Run("should not prepend the hub registry to the image name", func(t *testing.T) { + t.Run("non-hub image", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("quay.io/foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "quay.io/foo:latest" { + t.Errorf("expected quay.io/foo:latest, got %s", img) + } + }) + + t.Run("explicitly including docker.io", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("docker.io/foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "docker.io/foo:latest" { + t.Errorf("expected docker.io/foo:latest, got %s", img) + } + }) + + t.Run("explicitly including registry.hub.docker.com", func(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "my-registry") + defer resetConfigForTests() + + s := newPrependHubRegistry() + + img, err := s.Substitute("registry.hub.docker.com/foo:latest") + if err != nil { + t.Fatal(err) + } + + if img != "registry.hub.docker.com/foo:latest" { + t.Errorf("expected registry.hub.docker.com/foo:latest, got %s", img) + } + }) + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index d8d9ce5618..038a7b85d8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,8 @@ import ( "github.com/magiconair/properties" ) +const ReaperDefaultImage = "docker.io/testcontainers/ryuk:0.5.1" + var ( tcConfig Config tcConfigOnce *sync.Once = new(sync.Once) @@ -22,6 +24,7 @@ type Config struct { Host string `properties:"docker.host,default="` TLSVerify int `properties:"docker.tls.verify,default=0"` CertPath string `properties:"docker.cert.path,default="` + HubImageNamePrefix string `properties:"hub.image.name.prefix,default="` RyukDisabled bool `properties:"ryuk.disabled,default=false"` RyukPrivileged bool `properties:"ryuk.container.privileged,default=false"` RyukReconnectionTimeout time.Duration `properties:"ryuk.reconnection.timeout,default=10s"` @@ -67,6 +70,11 @@ func read() Config { config.RyukDisabled = ryukDisabledEnv == "true" } + hubImageNamePrefix := os.Getenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX") + if hubImageNamePrefix != "" { + config.HubImageNamePrefix = hubImageNamePrefix + } + ryukPrivilegedEnv := os.Getenv("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED") if parseBool(ryukPrivilegedEnv) { config.RyukPrivileged = ryukPrivilegedEnv == "true" diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2b48e65653..39cb0a87a7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -19,6 +19,7 @@ const ( // unset environment variables to avoid side effects // execute this function before each test func resetTestEnv(t *testing.T) { + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "") t.Setenv("TESTCONTAINERS_RYUK_DISABLED", "") t.Setenv("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "") } @@ -53,6 +54,8 @@ func TestReadConfig(t *testing.T) { func TestReadTCConfig(t *testing.T) { resetTestEnv(t) + const defaultHubPrefix string = "registry.mycompany.com/mirror" + t.Run("HOME is not set", func(t *testing.T) { t.Setenv("HOME", "") t.Setenv("USERPROFILE", "") // Windows support @@ -68,14 +71,16 @@ func TestReadTCConfig(t *testing.T) { t.Setenv("HOME", "") t.Setenv("USERPROFILE", "") // Windows support t.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", defaultHubPrefix) t.Setenv("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "true") config := read() expected := Config{ - RyukDisabled: true, - RyukPrivileged: true, - Host: "", // docker socket is empty at the properties file + HubImageNamePrefix: defaultHubPrefix, + RyukDisabled: true, + RyukPrivileged: true, + Host: "", // docker socket is empty at the properties file } assert.Equal(t, expected, config) @@ -110,12 +115,14 @@ func TestReadTCConfig(t *testing.T) { t.Setenv("HOME", tmpDir) t.Setenv("USERPROFILE", tmpDir) // Windows support t.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", defaultHubPrefix) t.Setenv("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "true") config := read() expected := Config{ - RyukDisabled: true, - RyukPrivileged: true, + HubImageNamePrefix: defaultHubPrefix, + RyukDisabled: true, + RyukPrivileged: true, } assert.Equal(t, expected, config) @@ -386,6 +393,40 @@ func TestReadTCConfig(t *testing.T) { }, defaultConfig, }, + { + "With Hub image name prefix set as a property", + `hub.image.name.prefix=` + defaultHubPrefix + `/props/`, + map[string]string{}, + Config{ + HubImageNamePrefix: defaultHubPrefix + "/props/", + RyukConnectionTimeout: defaultRyukConnectionTimeout, + RyukReconnectionTimeout: defaultRyukReonnectionTimeout, + }, + }, + { + "With Hub image name prefix set as env var", + ``, + map[string]string{ + "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX": defaultHubPrefix + "/env/", + }, + Config{ + HubImageNamePrefix: defaultHubPrefix + "/env/", + RyukConnectionTimeout: defaultRyukConnectionTimeout, + RyukReconnectionTimeout: defaultRyukReonnectionTimeout, + }, + }, + { + "With Hub image name prefix set as env var and properties: Env var wins", + `hub.image.name.prefix=` + defaultHubPrefix + `/props/`, + map[string]string{ + "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX": defaultHubPrefix + "/env/", + }, + Config{ + HubImageNamePrefix: defaultHubPrefix + "/env/", + RyukConnectionTimeout: defaultRyukConnectionTimeout, + RyukReconnectionTimeout: defaultRyukReonnectionTimeout, + }, + }, } for _, tt := range tests { t.Run(fmt.Sprintf(tt.name), func(t *testing.T) { diff --git a/mkdocs.yml b/mkdocs.yml index 5524330a4a..1a3f8a6863 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,10 +40,10 @@ nav: - Quickstart: quickstart.md - Features: - features/creating_container.md + - features/configuration.md - features/image_name_substitution.md - features/files_and_mounts.md - features/creating_networks.md - - features/configuration.md - features/networking.md - features/garbage_collector.md - features/build_from_dockerfile.md diff --git a/network.go b/network.go index c96fc11c92..82b4d19605 100644 --- a/network.go +++ b/network.go @@ -41,5 +41,5 @@ type NetworkRequest struct { SkipReaper bool // Deprecated: The reaper is globally controlled by the .testcontainers.properties file or the TESTCONTAINERS_RYUK_DISABLED environment variable ReaperImage string // Deprecated: use WithImageName ContainerOption instead. Alternative reaper registry - ReaperOptions []ContainerOption // Reaper options to use for this network + ReaperOptions []ContainerOption // Deprecated: the reaper is configured at the properties level, for an entire test session } diff --git a/options.go b/options.go index 093ef9ab3f..6ce0d94309 100644 --- a/options.go +++ b/options.go @@ -11,6 +11,8 @@ import ( "github.com/docker/docker/api/types/network" tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/internal/config" + "github.com/testcontainers/testcontainers-go/internal/testcontainersdocker" "github.com/testcontainers/testcontainers-go/wait" ) @@ -78,6 +80,51 @@ type ImageSubstitutor interface { // } +// prependHubRegistry represents a way to prepend a custom Hub registry to the image name, +// using the HubImageNamePrefix configuration value +type prependHubRegistry struct { + prefix string +} + +// newPrependHubRegistry creates a new prependHubRegistry +func newPrependHubRegistry() prependHubRegistry { + hubPrefix := config.Read().HubImageNamePrefix + + return prependHubRegistry{ + prefix: hubPrefix, + } +} + +// Description returns the name of the type and a short description of how it modifies the image. +func (p prependHubRegistry) Description() string { + return fmt.Sprintf("HubImageSubstitutor (prepends %s)", p.prefix) +} + +// Substitute prepends the Hub prefix to the image name, with certain conditions: +// - if the prefix is empty, the image is returned as is. +// - if the image is a non-hub image (e.g. where another registry is set), the image is returned as is. +// - if the image is a Docker Hub image where the hub registry is explicitly part of the name +// (i.e. anything with a docker.io or registry.hub.docker.com host part), the image is returned as is. +func (p prependHubRegistry) Substitute(image string) (string, error) { + registry := testcontainersdocker.ExtractRegistry(image, "") + + // add the exclusions in the right order + exclusions := []func() bool{ + func() bool { return p.prefix == "" }, // no prefix set at the configuration level + func() bool { return registry != "" }, // non-hub image + func() bool { return registry == "docker.io" }, // explicitly including docker.io + func() bool { return registry == "registry.hub.docker.com" }, // explicitly including registry.hub.docker.com + } + + for _, exclusion := range exclusions { + if exclusion() { + return image, nil + } + } + + return fmt.Sprintf("%s/%s", p.prefix, image), nil +} + // WithImageSubstitutors sets the image substitutors for a container func WithImageSubstitutors(fn ...ImageSubstitutor) CustomizeRequestOption { return func(req *GenericContainerRequest) { diff --git a/reaper.go b/reaper.go index 3521a8827d..72a5acab3f 100644 --- a/reaper.go +++ b/reaper.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/go-connections/nat" + "github.com/testcontainers/testcontainers-go/internal/config" "github.com/testcontainers/testcontainers-go/internal/testcontainersdocker" "github.com/testcontainers/testcontainers-go/wait" ) @@ -28,14 +29,14 @@ const ( TestcontainerLabelSessionID = TestcontainerLabel + ".sessionId" // Deprecated: it has been replaced by the internal testcontainersdocker.LabelReaper TestcontainerLabelIsReaper = TestcontainerLabel + ".reaper" - - ReaperDefaultImage = "docker.io/testcontainers/ryuk:0.5.1" ) var ( - reaperInstance *Reaper // We would like to create reaper only once - reaperMutex sync.Mutex - reaperOnce sync.Once + // Deprecated: it has been replaced by an internal value + ReaperDefaultImage = config.ReaperDefaultImage + reaperInstance *Reaper // We would like to create reaper only once + reaperMutex sync.Mutex + reaperOnce sync.Once ) // ReaperProvider represents a provider for the reaper to run itself with @@ -48,7 +49,7 @@ type ReaperProvider interface { // NewReaper creates a Reaper with a sessionID to identify containers and a provider to use // Deprecated: it's not possible to create a reaper anymore. func NewReaper(ctx context.Context, sessionID string, provider ReaperProvider, reaperImageName string) (*Reaper, error) { - return reuseOrCreateReaper(ctx, sessionID, provider, WithImageName(reaperImageName)) + return reuseOrCreateReaper(ctx, sessionID, provider) } // reaperContainerNameFromSessionID returns the container name that uniquely @@ -130,7 +131,7 @@ func lookUpReaperContainer(ctx context.Context, sessionID string) (*DockerContai // reuseOrCreateReaper returns an existing Reaper instance if it exists and is running. Otherwise, a new Reaper instance // will be created with a sessionID to identify containers in the same test session/program. -func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperProvider, opts ...ContainerOption) (*Reaper, error) { +func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperProvider) (*Reaper, error) { reaperMutex.Lock() defer reaperMutex.Unlock() @@ -168,7 +169,7 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP // synchronization primitive to avoid multiple executions of this function to create the reaper var reaperErr error reaperOnce.Do(func() { - r, err := newReaper(ctx, sessionID, provider, opts...) + r, err := newReaper(ctx, sessionID, provider) if err != nil { reaperErr = err return @@ -203,7 +204,7 @@ func reuseReaperContainer(ctx context.Context, sessionID string, provider Reaper // newReaper creates a Reaper with a sessionID to identify containers and a // provider to use. Do not call this directly, use reuseOrCreateReaper instead. -func newReaper(ctx context.Context, sessionID string, provider ReaperProvider, opts ...ContainerOption) (*Reaper, error) { +func newReaper(ctx context.Context, sessionID string, provider ReaperProvider) (*Reaper, error) { dockerHostMount := testcontainersdocker.ExtractDockerSocket(ctx) reaper := &Reaper{ @@ -215,20 +216,13 @@ func newReaper(ctx context.Context, sessionID string, provider ReaperProvider, o tcConfig := provider.Config().Config - reaperOpts := containerOptions{} - - for _, opt := range opts { - opt(&reaperOpts) - } - req := ContainerRequest{ - Image: reaperImage(reaperOpts.ImageName), - ExposedPorts: []string{string(listeningPort)}, - Labels: testcontainersdocker.DefaultLabels(sessionID), - Privileged: tcConfig.RyukPrivileged, - WaitingFor: wait.ForListeningPort(listeningPort), - Name: reaperContainerNameFromSessionID(sessionID), - ReaperOptions: opts, + Image: config.ReaperDefaultImage, + ExposedPorts: []string{string(listeningPort)}, + Labels: testcontainersdocker.DefaultLabels(sessionID), + Privileged: tcConfig.RyukPrivileged, + WaitingFor: wait.ForListeningPort(listeningPort), + Name: reaperContainerNameFromSessionID(sessionID), HostConfigModifier: func(hc *container.HostConfig) { hc.AutoRemove = true hc.Binds = []string{dockerHostMount + ":/var/run/docker.sock"} @@ -243,9 +237,6 @@ func newReaper(ctx context.Context, sessionID string, provider ReaperProvider, o req.Env["RYUK_RECONNECTION_TIMEOUT"] = to.String() } - // keep backwards compatibility - req.ReaperImage = req.Image - // include reaper-specific labels to the reaper container req.Labels[testcontainersdocker.LabelReaper] = "true" req.Labels[testcontainersdocker.LabelRyuk] = "true" @@ -376,10 +367,3 @@ func (r *Reaper) Labels() map[string]string { testcontainersdocker.LabelSessionID: r.SessionID, } } - -func reaperImage(reaperImageName string) string { - if reaperImageName == "" { - return ReaperDefaultImage - } - return reaperImageName -} diff --git a/reaper_test.go b/reaper_test.go index a1c91dbe5b..718491d700 100644 --- a/reaper_test.go +++ b/reaper_test.go @@ -3,6 +3,7 @@ package testcontainers import ( "context" "errors" + "os" "sync" "testing" "time" @@ -34,7 +35,9 @@ type mockReaperProvider struct { func newMockReaperProvider(t *testing.T) *mockReaperProvider { m := &mockReaperProvider{ - config: TestcontainersConfig{}, + config: TestcontainersConfig{ + Config: config.Config{}, + }, t: t, initialReaper: reaperInstance, //nolint:govet @@ -85,17 +88,13 @@ func (m *mockReaperProvider) Config() TestcontainersConfig { // createContainerRequest creates the expected request and allows for customization func createContainerRequest(customize func(ContainerRequest) ContainerRequest) ContainerRequest { req := ContainerRequest{ - Image: "reaperImage", - ReaperImage: "reaperImage", + Image: config.ReaperDefaultImage, ExposedPorts: []string{"8080/tcp"}, Labels: testcontainersdocker.DefaultLabels(testSessionID), HostConfigModifier: func(hostConfig *container.HostConfig) { hostConfig.Binds = []string{testcontainersdocker.ExtractDockerSocket(context.Background()) + ":/var/run/docker.sock"} }, WaitingFor: wait.ForListeningPort(nat.Port("8080/tcp")), - ReaperOptions: []ContainerOption{ - WithImageName("reaperImage"), - }, Env: map[string]string{ "RYUK_CONNECTION_TIMEOUT": "1m0s", "RYUK_RECONNECTION_TIMEOUT": "10s", @@ -316,6 +315,7 @@ func Test_NewReaper(t *testing.T) { req ContainerRequest config TestcontainersConfig ctx context.Context + env map[string]string } tests := []cases{ @@ -368,10 +368,51 @@ func Test_NewReaper(t *testing.T) { }}, ctx: context.WithValue(context.TODO(), testcontainersdocker.DockerHostContextKey, testcontainersdocker.DockerSocketPathWithSchema), }, + { + name: "Reaper including custom Hub prefix", + req: createContainerRequest(func(req ContainerRequest) ContainerRequest { + req.Image = config.ReaperDefaultImage + req.Privileged = true + return req + }), + config: TestcontainersConfig{Config: config.Config{ + HubImageNamePrefix: "registry.mycompany.com/mirror", + RyukPrivileged: true, + RyukConnectionTimeout: time.Minute, + RyukReconnectionTimeout: 10 * time.Second, + }}, + }, + { + name: "Reaper including custom Hub prefix as env var", + req: createContainerRequest(func(req ContainerRequest) ContainerRequest { + req.Image = config.ReaperDefaultImage + req.Privileged = true + return req + }), + config: TestcontainersConfig{Config: config.Config{ + RyukPrivileged: true, + RyukConnectionTimeout: time.Minute, + RyukReconnectionTimeout: 10 * time.Second, + }}, + env: map[string]string{ + "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX": "registry.mycompany.com/mirror", + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + if test.env != nil { + config.Reset() // reset the config using the internal method to avoid the sync.Once + for k, v := range test.env { + t.Setenv(k, v) + } + } + + if prefix := os.Getenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX"); prefix != "" { + test.config.Config.HubImageNamePrefix = prefix + } + provider := newMockReaperProvider(t) provider.config = test.config t.Cleanup(provider.RestoreReaperState) @@ -380,7 +421,7 @@ func Test_NewReaper(t *testing.T) { test.ctx = context.TODO() } - _, err := reuseOrCreateReaper(test.ctx, testSessionID, provider, test.req.ReaperOptions...) + _, err := reuseOrCreateReaper(test.ctx, testSessionID, provider) // we should have errored out see mockReaperProvider.RunContainer assert.EqualError(t, err, "expected") @@ -398,31 +439,6 @@ func Test_NewReaper(t *testing.T) { } } -func Test_ReaperForNetwork(t *testing.T) { - provider := newMockReaperProvider(t) - t.Cleanup(provider.RestoreReaperState) - - ctx := context.Background() - - networkName := "test-network-with-custom-reaper" - - req := GenericNetworkRequest{ - NetworkRequest: NetworkRequest{ - Name: networkName, - CheckDuplicate: true, - ReaperOptions: []ContainerOption{ - WithImageName("reaperImage"), - }, - }, - } - - _, err := reuseOrCreateReaper(ctx, testSessionID, provider, req.ReaperOptions...) - assert.EqualError(t, err, "expected") - - assert.Equal(t, "reaperImage", provider.req.Image) - assert.Equal(t, "reaperImage", provider.req.ReaperImage) -} - func Test_ReaperReusedIfHealthy(t *testing.T) { config.Reset() // reset the config using the internal method to avoid the sync.Once tcConfig := config.Read() diff --git a/scripts/bump-reaper.sh b/scripts/bump-reaper.sh index 61a231d190..4e1211a1b3 100755 --- a/scripts/bump-reaper.sh +++ b/scripts/bump-reaper.sh @@ -14,7 +14,7 @@ readonly CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" readonly DRY_RUN="${DRY_RUN:-true}" readonly ROOT_DIR="$(dirname "$CURRENT_DIR")" -readonly REAPER_FILE="${ROOT_DIR}/reaper.go" +readonly REAPER_FILE="${ROOT_DIR}/internal/config/config.go" function main() { echo "Updating Ryuk version:" @@ -26,12 +26,12 @@ function main() { local escapedRyukVersion="${ryukVersion//\//\\/}" echo " - New: ${ryukVersion}" - # Bump the version in the version.go file + # Bump the version in the config.go file if [[ "${DRY_RUN}" == "true" ]]; then echo "sed \"s/ReaperDefaultImage = \".*\"/ReaperDefaultImage = \"${escapedRyukVersion}\"/g\" ${REAPER_FILE} > ${REAPER_FILE}.tmp" echo "mv ${REAPER_FILE}.tmp ${REAPER_FILE}" else - # replace using sed the version in the reaper.go file + # replace using sed the version in the config.go file sed "s/ReaperDefaultImage = \".*\"/ReaperDefaultImage = \"${escapedRyukVersion}\"/g" ${REAPER_FILE} > ${REAPER_FILE}.tmp mv ${REAPER_FILE}.tmp ${REAPER_FILE} fi @@ -39,7 +39,7 @@ function main() { # This function reads the reaper.go file and extracts the current version. function extractCurrentVersion() { - cat "${REAPER_FILE}" | grep 'ReaperDefaultImage = "' | cut -d '"' -f 2 + cat "${REAPER_FILE}" | grep 'ReaperDefaultImage = ' | cut -d '=' -f 2 | cut -d '"' -f 1 } main "$@"