diff --git a/.travis.yml b/.travis.yml index f68292186c..e19669cc7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,21 @@ language: go os: linux - +dist: bionic +env: + - IMAGE_REPO=localhost:5000 go: - "1.13.3" go_import_path: github.com/GoogleContainerTools/kaniko +before_install: + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff + - docker run -d -p 5000:5000 --restart always --name registry registry:2 + - ./integration/replace-gcr-with-local-registry.sh integration/dockerfiles script: - make test + - ./integration-test.sh --uploadToGCS=false - make images diff --git a/integration/dockerfiles/Dockerfile_onbuild_base b/integration/dockerfiles/Dockerfile_onbuild_base index eebdf48613..aaf928b79a 100644 --- a/integration/dockerfiles/Dockerfile_onbuild_base +++ b/integration/dockerfiles/Dockerfile_onbuild_base @@ -1,4 +1,4 @@ -FROM marketplace.gcr.io/google/ubuntu1804@sha256:4649ae6b381090fba6db38137eb05e03f44bf43c40149f734241c9f96aa0e001 +FROM ubuntu:18.04 ENV dir /tmp/dir/ ONBUILD RUN echo "onbuild" > /tmp/onbuild ONBUILD RUN mkdir $dir diff --git a/integration/dockerfiles/Dockerfile_test_add b/integration/dockerfiles/Dockerfile_test_add index e5d2ca3bcb..dfb748d8aa 100644 --- a/integration/dockerfiles/Dockerfile_test_add +++ b/integration/dockerfiles/Dockerfile_test_add @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 # First, try adding some regular files ADD context/foo foo ADD context/foo /foodir/ diff --git a/integration/dockerfiles/Dockerfile_test_cache b/integration/dockerfiles/Dockerfile_test_cache index e3ebe304dc..d6f78798c3 100644 --- a/integration/dockerfiles/Dockerfile_test_cache +++ b/integration/dockerfiles/Dockerfile_test_cache @@ -16,7 +16,7 @@ # If the image is built twice, /date should be the same in both images # if the cache is implemented correctly -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 RUN date > /date COPY context/foo /foo RUN echo hey diff --git a/integration/dockerfiles/Dockerfile_test_cache_install b/integration/dockerfiles/Dockerfile_test_cache_install index b93eb4ad50..455fc26c09 100644 --- a/integration/dockerfiles/Dockerfile_test_cache_install +++ b/integration/dockerfiles/Dockerfile_test_cache_install @@ -16,7 +16,7 @@ # /date should be the same regardless of when this image is built # if the cache is implemented correctly -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 WORKDIR /foo RUN apt-get update && apt-get install -y make COPY context/bar /context diff --git a/integration/dockerfiles/Dockerfile_test_cmd b/integration/dockerfiles/Dockerfile_test_cmd index a2f0162b0a..9f71cfc543 100644 --- a/integration/dockerfiles/Dockerfile_test_cmd +++ b/integration/dockerfiles/Dockerfile_test_cmd @@ -1,5 +1,4 @@ -FROM gcr.io/distroless/base@sha256:628939ac8bf3f49571d05c6c76b8688cb4a851af6c7088e599388259875bde20 AS first +FROM debian:10.2 AS first CMD ["mycmd"] - FROM first ENTRYPOINT ["myentrypoint"] # This should clear out CMD in the config metadata diff --git a/integration/dockerfiles/Dockerfile_test_env b/integration/dockerfiles/Dockerfile_test_env index 5083a8907a..3f90703331 100644 --- a/integration/dockerfiles/Dockerfile_test_env +++ b/integration/dockerfiles/Dockerfile_test_env @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 ENV hey hey ENV PATH /usr/local ENV testmultipleeq="this=is a=test string=with a=lot of=equals" diff --git a/integration/dockerfiles/Dockerfile_test_expose b/integration/dockerfiles/Dockerfile_test_expose index 7df9f63369..ec25c7db9e 100644 --- a/integration/dockerfiles/Dockerfile_test_expose +++ b/integration/dockerfiles/Dockerfile_test_expose @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 EXPOSE 80 EXPOSE 81/udp ENV protocol tcp diff --git a/integration/dockerfiles/Dockerfile_test_extract_fs b/integration/dockerfiles/Dockerfile_test_extract_fs index 64cf5576a1..2a29339e07 100644 --- a/integration/dockerfiles/Dockerfile_test_extract_fs +++ b/integration/dockerfiles/Dockerfile_test_extract_fs @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 diff --git a/integration/dockerfiles/Dockerfile_test_label b/integration/dockerfiles/Dockerfile_test_label index 8c825dbcb0..1412ac3574 100644 --- a/integration/dockerfiles/Dockerfile_test_label +++ b/integration/dockerfiles/Dockerfile_test_label @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 LABEL foo=bar LABEL "baz"="bat" ENV label1 "mylabel" diff --git a/integration/dockerfiles/Dockerfile_test_meta_arg b/integration/dockerfiles/Dockerfile_test_meta_arg index b90662138c..408fc546a3 100644 --- a/integration/dockerfiles/Dockerfile_test_meta_arg +++ b/integration/dockerfiles/Dockerfile_test_meta_arg @@ -1,15 +1,16 @@ -ARG REGISTRY=gcr.io -ARG REPO=google-appengine +ARG REGISTRY=docker.io +ARG IMAGE=debian +ARG TAG=9.11 ARG WORD=hello ARG W0RD2=hey -FROM ${REGISTRY}/${REPO}/debian9 as stage1 +FROM ${REGISTRY}/${IMAGE}:${TAG} as stage1 # Should evaluate WORD and create /tmp/hello ARG WORD RUN touch /${WORD} -FROM ${REGISTRY}/${REPO}/debian9 +FROM ${REGISTRY}/${IMAGE}:${TAG} COPY --from=stage1 /hello /tmp diff --git a/integration/dockerfiles/Dockerfile_test_metadata b/integration/dockerfiles/Dockerfile_test_metadata index 00c325f2c0..7abeacdd57 100644 --- a/integration/dockerfiles/Dockerfile_test_metadata +++ b/integration/dockerfiles/Dockerfile_test_metadata @@ -1,4 +1,4 @@ -FROM gcr.io/distroless/base@sha256:628939ac8bf3f49571d05c6c76b8688cb4a851af6c7088e599388259875bde20 +FROM debian:10.2 CMD ["command", "one"] CMD ["command", "two"] CMD echo "hello" diff --git a/integration/dockerfiles/Dockerfile_test_multistage b/integration/dockerfiles/Dockerfile_test_multistage index 00325f26c6..df8f05aa52 100644 --- a/integration/dockerfiles/Dockerfile_test_multistage +++ b/integration/dockerfiles/Dockerfile_test_multistage @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:f0159d14385afcb58a9b2fa8955c0cb64bd3abc365e8589f8c2dd38150fbfdbe as base +FROM debian:9.11 as base COPY . . FROM scratch as second @@ -20,4 +20,4 @@ FROM fedora@sha256:c4cc32b09c6ae3f1353e7e33a8dda93dc41676b923d6d89afa996b421cc5a FROM fourth ARG file COPY --from=second /foo ${file} -COPY --from=gcr.io/google-appengine/debian9@sha256:00109fa40230a081f5ecffe0e814725042ff62a03e2d1eae0563f1f82eaeae9b /etc/os-release /new +COPY --from=debian:9.11 /etc/os-release /new diff --git a/integration/dockerfiles/Dockerfile_test_run b/integration/dockerfiles/Dockerfile_test_run index 61db07c023..9891a43818 100644 --- a/integration/dockerfiles/Dockerfile_test_run +++ b/integration/dockerfiles/Dockerfile_test_run @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 RUN echo "hey" > /etc/foo RUN echo "baz" > /etc/baz RUN cp /etc/baz /etc/bar diff --git a/integration/dockerfiles/Dockerfile_test_run_2 b/integration/dockerfiles/Dockerfile_test_run_2 index 8e685de6b3..0087dd6ea9 100644 --- a/integration/dockerfiles/Dockerfile_test_run_2 +++ b/integration/dockerfiles/Dockerfile_test_run_2 @@ -15,6 +15,6 @@ # Test to make sure the executor builds an image correctly # when no files are changed -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 RUN echo "hey" MAINTAINER kaniko diff --git a/integration/dockerfiles/Dockerfile_test_user b/integration/dockerfiles/Dockerfile_test_user index 51896647fb..a017b0d6ee 100644 --- a/integration/dockerfiles/Dockerfile_test_user +++ b/integration/dockerfiles/Dockerfile_test_user @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 USER testuser:testgroup diff --git a/integration/dockerfiles/Dockerfile_test_user_run b/integration/dockerfiles/Dockerfile_test_user_run index 3732e625b9..c6ae89abcb 100644 --- a/integration/dockerfiles/Dockerfile_test_user_run +++ b/integration/dockerfiles/Dockerfile_test_user_run @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 RUN useradd testuser RUN groupadd testgroup USER testuser:testgroup diff --git a/integration/dockerfiles/Dockerfile_test_volume b/integration/dockerfiles/Dockerfile_test_volume index a4db420acb..13e9cb5e63 100644 --- a/integration/dockerfiles/Dockerfile_test_volume +++ b/integration/dockerfiles/Dockerfile_test_volume @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 RUN mkdir /foo RUN echo "hello" > /foo/hey VOLUME /foo/bar /tmp /qux/quux diff --git a/integration/dockerfiles/Dockerfile_test_volume_2 b/integration/dockerfiles/Dockerfile_test_volume_2 index 77a116a082..40180e0f3c 100644 --- a/integration/dockerfiles/Dockerfile_test_volume_2 +++ b/integration/dockerfiles/Dockerfile_test_volume_2 @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +FROM debian:9.11 VOLUME /foo1 RUN echo "hello" > /foo1/hello WORKDIR /foo1/bar diff --git a/integration/dockerfiles/Dockerfile_test_workdir b/integration/dockerfiles/Dockerfile_test_workdir index 9c2794f702..df8bd3e4b7 100644 --- a/integration/dockerfiles/Dockerfile_test_workdir +++ b/integration/dockerfiles/Dockerfile_test_workdir @@ -1,4 +1,4 @@ -FROM gcr.io/google-appengine/debian9@sha256:6b3aa04751aa2ac3b0c7be4ee71148b66d693ad212ce6d3244bd2a2a147f314a +FROM debian:9.11 COPY context/foo foo WORKDIR /test # Test that this will be appended on to the previous command, to create /test/workdir diff --git a/integration/images.go b/integration/images.go index f98a716d42..f4df7af494 100644 --- a/integration/images.go +++ b/integration/images.go @@ -220,8 +220,8 @@ func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, docke additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) - dockerRunFlags := []string{ - "run", "-e", benchmarkEnv, + dockerRunFlags := []string{"run", "--net=host", + "-e", benchmarkEnv, "-v", cwd + ":/workspace", "-v", benchmarkDir + ":/kaniko/benchmarks", } @@ -251,7 +251,7 @@ func populateVolumeCache() error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) warmerCmd := exec.Command("docker", - append([]string{"run", + append([]string{"run", "--net=host", "-d", "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", "-v", cwd + ":/workspace", @@ -284,7 +284,7 @@ func (d *DockerFileBuilder) buildCachedImages(config *gcpConfig, cacheRepo, dock } kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version) - dockerRunFlags := []string{"run", + dockerRunFlags := []string{"run", "--net=host", "-v", cwd + ":/workspace", "-e", benchmarkEnv} dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) @@ -315,7 +315,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi buildContextPath := "./relative-subdirectory" kanikoImage := GetKanikoImage(imageRepo, dockerfile) - dockerRunFlags := []string{"run", "-v", cwd + ":/workspace"} + dockerRunFlags := []string{"run", "--net=host", "-v", cwd + ":/workspace"} dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) dockerRunFlags = append(dockerRunFlags, ExecutorImage, "-f", dockerfile, diff --git a/integration/integration_test.go b/integration/integration_test.go index 43b51fbe71..c5c910d885 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -88,26 +88,28 @@ func TestMain(m *testing.M) { } config = initGCPConfig() - contextFile, err := CreateIntegrationTarball() - if err != nil { - fmt.Println("Failed to create tarball of integration files for build context", err) - os.Exit(1) - } + if config.uploadToGCS { + contextFile, err := CreateIntegrationTarball() + if err != nil { + fmt.Println("Failed to create tarball of integration files for build context", err) + os.Exit(1) + } - fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile) - if err != nil { - fmt.Println("Failed to upload build context", err) - os.Exit(1) - } + fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile) + if err != nil { + fmt.Println("Failed to upload build context", err) + os.Exit(1) + } - err = os.Remove(contextFile) - if err != nil { - fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err) - os.Exit(1) - } + err = os.Remove(contextFile) + if err != nil { + fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err) + os.Exit(1) + } - RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) }) - defer DeleteFromBucket(fileInBucket) + RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) }) + defer DeleteFromBucket(fileInBucket) + } setupCommands := []struct { name string @@ -197,52 +199,25 @@ func TestRun(t *testing.T) { } } -func TestGitBuildcontext(t *testing.T) { - repo := "github.com/GoogleContainerTools/kaniko" - dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" - - // Build with docker - dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") - dockerCmd := exec.Command("docker", - append([]string{"build", - "-t", dockerImage, - "-f", dockerfile, - repo})...) - out, err := RunCommandWithoutTest(dockerCmd) - if err != nil { - t.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) - } - - // Build with kaniko - kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") - dockerRunFlags := []string{"run"} - dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) - dockerRunFlags = append(dockerRunFlags, ExecutorImage, - "-f", dockerfile, - "-d", kanikoImage, - "-c", fmt.Sprintf("git://%s", repo)) - - kanikoCmd := exec.Command("docker", dockerRunFlags...) - - out, err = RunCommandWithoutTest(kanikoCmd) - if err != nil { - t.Errorf("Failed to build image %s with kaniko command \"%s\": %v %s", dockerImage, kanikoCmd.Args, err, string(out)) +func getGitRepo() string { + var branch, repoSlug string + if _, ok := os.LookupEnv("TRAVIS"); ok { + if os.Getenv("TRAVIS_PULL_REQUEST") != "false" { + branch = os.Getenv("TRAVIS_PULL_REQUEST_BRANCH") + repoSlug = os.Getenv("TRAVIS_PULL_REQUEST_SLUG") + log.Printf("Travis CI Pull request source repo: %s branch: %s\n", repoSlug, branch) + } else { + branch = os.Getenv("TRAVIS_BRANCH") + repoSlug = os.Getenv("TRAVIS_REPO_SLUG") + log.Printf("Travis CI pepo: %s branch: %s\n", repoSlug, branch) + } + return "github.com/" + repoSlug + "#refs/heads/" + branch } - - // container-diff - daemonDockerImage := daemonPrefix + dockerImage - containerdiffCmd := exec.Command("container-diff", "diff", "--no-cache", - daemonDockerImage, kanikoImage, - "-q", "--type=file", "--type=metadata", "--json") - diff := RunCommand(containerdiffCmd, t) - t.Logf("diff = %s", string(diff)) - - expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) - checkContainerDiffOutput(t, diff, expected) + return "github.com/GoogleContainerTools/kaniko" } -func TestGitBuildContextWithBranch(t *testing.T) { - repo := "github.com/GoogleContainerTools/kaniko#refs/tags/v0.10.0" +func TestGitBuildcontext(t *testing.T) { + repo := getGitRepo() dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" // Build with docker @@ -259,7 +234,7 @@ func TestGitBuildContextWithBranch(t *testing.T) { // Build with kaniko kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") - dockerRunFlags := []string{"run"} + dockerRunFlags := []string{"run", "--net=host"} dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) dockerRunFlags = append(dockerRunFlags, ExecutorImage, "-f", dockerfile, @@ -567,6 +542,7 @@ type gcpConfig struct { hardlinkBaseImage string serviceAccount string dockerMajorVersion int + uploadToGCS bool } type imageDetails struct { @@ -584,6 +560,7 @@ func initGCPConfig() *gcpConfig { flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.") flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.") + flag.BoolVar(&c.uploadToGCS, "uploadToGCS", true, "Upload the tar-ed contents of `integration` dir to GCS bucket. Default is true. Set this to false to prevent uploading.") flag.Parse() if len(c.serviceAccount) > 0 { diff --git a/integration/replace-gcr-with-local-registry.sh b/integration/replace-gcr-with-local-registry.sh new file mode 100755 index 0000000000..e40337fdee --- /dev/null +++ b/integration/replace-gcr-with-local-registry.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script is needed due to the following bug: +# https://github.com/GoogleContainerTools/kaniko/issues/966 + + +if [ "$#" -ne 1 ]; then + echo "Please specify path to dockerfiles as first argument." + echo "Usage: `basename $0` integration/dockerfiles" + exit 2 +fi + +dir_with_docker_files=$1 + +for dockerfile in $dir_with_docker_files/*; do + cat $dockerfile | grep '^FROM' | grep "gcr" | while read -r line; do + gcr_repo=$(echo "$line" | awk '{ print $2 }') + local_repo=$(echo "$gcr_repo" | sed -e "s/^.*gcr.io\(\/.*\)$/localhost:5000\1/") + remove_digest=$(echo "$local_repo" | cut -f1 -d"@") + echo "Running docker pull $gcr_repo" + docker pull "$gcr_repo" + echo "Running docker tag $gcr_repo $remove_digest" + docker tag "$gcr_repo" "$remove_digest" + echo "Running docker push $remove_digest" + docker push "$remove_digest" + echo "Updating dockerfile $dockerfile to use local repo $local_repo" + sed -i -e "s/^\(FROM \).*gcr.io\(.*\)$/\1localhost:5000\2/" $dockerfile + done +done