Skip to content

Commit

Permalink
Let Kaniko E2E test work with a KO_DOCKER_REPO env
Browse files Browse the repository at this point in the history 
If test runner set a KO_DOCKER_REPO variable, use it, so we run the tests
against an external container registry. If not set, the kaniko tests
will spin up its own local registry. Bring back the GCP secret support
so this works in the existing CI, but also supports the case where
KO_DOCKER_REPO points to a container registry where no secret required,
like in the kind based tests.

Add a "test_setup" function to the e2e script, which is picked up after
the setup of the cluster, which provisions a GCP service account to be
used by tests via the GCP_SERVICE_ACCOUNT_KEY_PATH env variable.

Since the local registry runs on HTTP (not HTTPS), the local registry
approach does not work for the helm test as the kubelet tries to use
HTTPS. If KO_DOCKER_REPO is not specified we either fail or skip the
test (no change in behaviour) depending on the value of missingKoFatal

This setup makes it easier to run E2E tests in different environments,
where a registry may or may not be available.

Signed-off-by: Andrea Frittoli <andrea.frittoli@uk.ibm.com>
  • Loading branch information
@afrittoli
afrittoli committed Dec 3, 2021
1 parent df5cc01 commit cdec71d
Show file tree
Hide file tree
Showing 6 changed files with 180 additions and 41 deletions.
29 changes: 23 additions & 6 deletions test/e2e-tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,23 @@

source $(git rev-parse --show-toplevel)/test/e2e-common.sh

# Script entry point.
function test_setup() {
# Setup a service account for kaniko and gcs / storage artifact tests
gcloud config set project "${E2E_PROJECT_ID}"

initialize $@
# create the service account
ACCOUNT_NAME=${E2E_PROJECT_ID}
gcloud iam service-accounts create $ACCOUNT_NAME --display-name $ACCOUNT_NAME
EMAIL=$(gcloud iam service-accounts list | grep $ACCOUNT_NAME | awk '{print $2}')

header "Setting up environment"
# add the storage.admin policy to the account so it can push containers
gcloud projects add-iam-policy-binding ${E2E_PROJECT_ID} --member serviceAccount:$EMAIL --role roles/storage.admin

install_pipeline_crd
# create the JSON key
gcloud iam service-accounts keys create config.json --iam-account $EMAIL

failed=0
export GCP_SERVICE_ACCOUNT_KEY_PATH="$PWD/config.json"
}

function set_feature_gate() {
local gate="$1"
Expand All @@ -52,6 +60,16 @@ function run_e2e() {
go_test_e2e -tags=examples -timeout=20m ./test/ || failed=1
}

# Script entry point.

initialize $@

header "Setting up environment"

install_pipeline_crd

failed=0

if [ "$PIPELINE_FEATURE_GATE" == "" ]; then
set_feature_gate "stable"
run_e2e
Expand All @@ -60,6 +78,5 @@ else
run_e2e
fi


(( failed )) && fail_test
success
4 changes: 3 additions & 1 deletion test/helm_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,12 @@ var (
// TestHelmDeployPipelineRun is an integration test that will verify a pipeline build an image
// and then using helm to deploy it
func TestHelmDeployPipelineRun(t *testing.T) {
repo := ensureDockerRepo(t)
repo := ensureDockerRepo(t, "helmtasktest")

ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
defer cancel()

c, namespace := setup(ctx, t)
setupClusterBindingForHelm(ctx, c, t, namespace)

Expand Down
67 changes: 54 additions & 13 deletions test/kaniko_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,10 @@ import (
"strings"
"testing"

"github.com/tektoncd/pipeline/test/parse"

"github.com/google/go-cmp/cmp"
"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
"github.com/tektoncd/pipeline/test/parse"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
knativetest "knative.dev/pkg/test"
Expand All @@ -45,6 +44,13 @@ const (

// TestTaskRun is an integration test that will verify a TaskRun using kaniko
func TestKanikoTaskRun(t *testing.T) {
var (
namespace, repo string
c *clients
err error
)
hasSecretConfig := false

ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
Expand All @@ -53,11 +59,23 @@ func TestKanikoTaskRun(t *testing.T) {
t.Skip("Skip test as skipRootUserTests set to true")
}

c, namespace := setup(ctx, t, withRegistry)
repo, err = getDockerRepo("kanikotasktest")
if err != nil {
// No KO_DOCKER_REPO set, use a sidecar registry instead
c, namespace = setup(ctx, t, withRegistry)
repo = fmt.Sprintf("registry.%s.svc.cluster.local:5000/kanikotasktest", namespace)
} else {
c, namespace = setup(ctx, t)
// When KO_DOCKER_REPO is set, we might need a secret attached to the service account
// If CreateGCPServiceAccountSecret no secret has been requested
// If CreateGCPServiceAccountSecret returns an error, a secret was requested, but failed
hasSecretConfig, err = CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret")
if err != nil {
t.Fatalf("Failed to create kaniko creds: %v", err)
}
}
t.Parallel()

repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace)

knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf)
defer tearDown(ctx, t, c, namespace)

Expand All @@ -72,7 +90,7 @@ func TestKanikoTaskRun(t *testing.T) {
}

t.Logf("Creating Task %s", kanikoTaskName)
if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace), metav1.CreateOptions{}); err != nil {
if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace, hasSecretConfig), metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err)
}

Expand Down Expand Up @@ -158,8 +176,8 @@ spec:
`, kanikoImageResourceName, repo))
}

func getTask(t *testing.T, repo, namespace string) *v1beta1.Task {
return parse.MustParseTask(t, fmt.Sprintf(`
func getTask(t *testing.T, repo, namespace string, withSecretConfig bool) *v1beta1.Task {
task := parse.MustParseTask(t, fmt.Sprintf(`
metadata:
name: %s
namespace: %s
Expand All @@ -177,16 +195,39 @@ spec:
args: ['--dockerfile=/workspace/gitsource/integration/dockerfiles/Dockerfile_test_label',
'--destination=%s',
'--context=/workspace/gitsource',
'--oci-layout-path=/workspace/output/builtImage',
'--insecure',
'--insecure-pull',
'--insecure-registry=registry.%s:5000/']
'--oci-layout-path=/workspace/output/builtImage']
securityContext:
runAsUser: 0
sidecars:
- name: registry
image: %s
`, kanikoTaskName, namespace, getTestImage(kanikoImage), repo, namespace, getTestImage(registryImage)))
`, kanikoTaskName, namespace, getTestImage(kanikoImage), repo, getTestImage(registryImage)))
if withSecretConfig {
// Mount an extra volume to the kaniko step and set the GOOGLE_APPLICATION_CREDENTIALS env
task.Spec.Volumes = []corev1.Volume{{
Name: "kaniko-secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "kaniko-secret",
},
},
}}
task.Spec.Steps[0].Env = append(
task.Spec.Steps[0].Env,
corev1.EnvVar{
Name: "GOOGLE_APPLICATION_CREDENTIALS",
Value: "/secrets/config.json",
},
)
task.Spec.Steps[0].VolumeMounts = append(
task.Spec.Steps[0].VolumeMounts,
corev1.VolumeMount{
Name: "kaniko-secret",
MountPath: "/secrets",
},
)
}
return task
}

func getTaskRun(t *testing.T, namespace string) *v1beta1.TaskRun {
Expand Down
14 changes: 7 additions & 7 deletions test/ko_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,24 @@ var (
missingKoFatal = "true"
)

func ensureDockerRepo(t *testing.T) string {
repo, err := getDockerRepo()
func ensureDockerRepo(t *testing.T, name string) string {
repo, err := getDockerRepo(name)
if err != nil {
if missingKoFatal == "false" {
t.Skip("KO_DOCKER_REPO env variable is required")
t.Skip(fmt.Sprintf("KO_DOCKER_REPO env variable is required for %s", name))
}
t.Fatal("KO_DOCKER_REPO env variable is required")
t.Fatal(fmt.Sprintf("KO_DOCKER_REPO env variable is required for %s", name))
}
return repo
}

func getDockerRepo() (string, error) {
func getDockerRepo(name string) (string, error) {
// according to knative/test-infra readme (https://github.com/knative/test-infra/blob/13055d769cc5e1756e605fcb3bcc1c25376699f1/scripts/README.md)
// the KO_DOCKER_REPO will be set with according to the project where the cluster is created
// it is used here to dynamically get the docker registry to push the image to
dockerRepo := os.Getenv("KO_DOCKER_REPO")
if dockerRepo == "" {
return "", errors.New("KO_DOCKER_REPO env variable is required")
return "", errors.New("KO_DOCKER_REPO env variable is not set")
}
return fmt.Sprintf("%s/kanikotasktest", dockerRepo), nil
return fmt.Sprintf("%s/%s", dockerRepo, name), nil
}
71 changes: 57 additions & 14 deletions test/v1alpha1/kaniko_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,37 @@ const (

// TestTaskRun is an integration test that will verify a TaskRun using kaniko
func TestKanikoTaskRun(t *testing.T) {
if skipRootUserTests {
t.Skip("Skip test as skipRootUserTests set to true")
}
var (
namespace, repo string
c *clients
err error
)
hasSecretConfig := false

ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
c, namespace := setup(ctx, t, withRegistry)
t.Parallel()

repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace)
if skipRootUserTests {
t.Skip("Skip test as skipRootUserTests set to true")
}

repo, err = getDockerRepo("kanikotasktest")
if err != nil {
// No KO_DOCKER_REPO set, use a sidecar registry instead
c, namespace = setup(ctx, t, withRegistry)
repo = fmt.Sprintf("registry.%s.svc.cluster.local:5000/kanikotasktest", namespace)
} else {
c, namespace = setup(ctx, t)
// When KO_DOCKER_REPO is set, we might need a secret attached to the service account
// If CreateGCPServiceAccountSecret no secret has been requested
// If CreateGCPServiceAccountSecret returns an error, a secret was requested, but failed
hasSecretConfig, err = CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret")
if err != nil {
t.Fatalf("Failed to create kaniko creds: %v", err)
}
}
t.Parallel()

knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf)
defer tearDown(ctx, t, c, namespace)
Expand All @@ -69,7 +89,7 @@ func TestKanikoTaskRun(t *testing.T) {
}

t.Logf("Creating Task %s", kanikoTaskName)
if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace), metav1.CreateOptions{}); err != nil {
if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace, hasSecretConfig), metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err)
}

Expand Down Expand Up @@ -152,8 +172,8 @@ spec:
`, kanikoImageResourceName, repo))
}

func getTask(t *testing.T, repo, namespace string) *v1alpha1.Task {
return parse.MustParseAlphaTask(t, fmt.Sprintf(`
func getTask(t *testing.T, repo, namespace string, withSecretConfig bool) *v1alpha1.Task {
task := parse.MustParseAlphaTask(t, fmt.Sprintf(`
metadata:
name: %s
spec:
Expand All @@ -171,16 +191,39 @@ spec:
args: ['--dockerfile=/workspace/gitsource/integration/dockerfiles/Dockerfile_test_label',
'--destination=%s',
'--context=/workspace/gitsource',
'--oci-layout-path=/workspace/output/builtImage',
'--insecure',
'--insecure-pull',
'--insecure-registry=registry.%s:5000/']
'--oci-layout-path=/workspace/output/builtImage']
securityContext:
runAsUser: 0
sidecars:
- name: registry
image: registry
`, kanikoTaskName, repo, namespace))
`, kanikoTaskName, repo))
if withSecretConfig {
// Mount an extra volume to the kaniko step and set the GOOGLE_APPLICATION_CREDENTIALS env
task.Spec.Volumes = []corev1.Volume{{
Name: "kaniko-secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "kaniko-secret",
},
},
}}
task.Spec.Steps[0].Env = append(
task.Spec.Steps[0].Env,
corev1.EnvVar{
Name: "GOOGLE_APPLICATION_CREDENTIALS",
Value: "/secrets/config.json",
},
)
task.Spec.Steps[0].VolumeMounts = append(
task.Spec.Steps[0].VolumeMounts,
corev1.VolumeMount{
Name: "kaniko-secret",
MountPath: "/secrets",
},
)
}
return task
}

func getTaskRun(t *testing.T, namespace string) *v1alpha1.TaskRun {
Expand Down
36 changes: 36 additions & 0 deletions test/v1alpha1/ko_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// +build e2e

/*
Copyright 2019 The Tekton Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
"errors"
"fmt"
"os"
)

func getDockerRepo(name string) (string, error) {
// according to knative/test-infra readme (https://github.com/knative/test-infra/blob/13055d769cc5e1756e605fcb3bcc1c25376699f1/scripts/README.md)
// the KO_DOCKER_REPO will be set with according to the project where the cluster is created
// it is used here to dynamically get the docker registry to push the image to
dockerRepo := os.Getenv("KO_DOCKER_REPO")
if dockerRepo == "" {
return "", errors.New("KO_DOCKER_REPO env variable is not set")
}
return fmt.Sprintf("%s/%s", dockerRepo, name), nil
}

0 comments on commit cdec71d

Please sign in to comment.