Let Kaniko E2E test work with a KO_DOCKER_REPO env

If test runner set a KO_DOCKER_REPO variable, use it, so we run the tests against an external container registry. If not set, the kaniko tests will spin up its own local registry. Bring back the GCP secret support so this works in the existing CI, but also supports the case where KO_DOCKER_REPO points to a container registry where no secret required, like in the kind based tests. Since the local registry runs on HTTP (not HTTPS), the local registry approach does not work for the helm test as the kubelet tries to use HTTPS. If KO_DOCKER_REPO is not specified we either fail or skip the test (no change in behaviour) depending on the value of missingKoFatal This setup makes it easier to run E2E tests in different environments, where a registry may or may not be available. Signed-off-by: Andrea Frittoli <andrea.frittoli@uk.ibm.com>
tektoncd · Dec 2, 2021 · 88681e9 · 88681e9
1 parent df5cc01
commit 88681e9
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 35 deletions.
diff --git a/test/helm_task_test.go b/test/helm_task_test.go
@@ -41,10 +41,12 @@ var (
 // TestHelmDeployPipelineRun is an integration test that will verify a pipeline build an image
 // and then using helm to deploy it
 func TestHelmDeployPipelineRun(t *testing.T) {
-	repo := ensureDockerRepo(t)
+	repo := ensureDockerRepo(t, "helmtasktest")
+
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
+
 	c, namespace := setup(ctx, t)
 	setupClusterBindingForHelm(ctx, c, t, namespace)
 

diff --git a/test/kaniko_task_test.go b/test/kaniko_task_test.go
@@ -24,11 +24,10 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/tektoncd/pipeline/test/parse"
-
 	"github.com/google/go-cmp/cmp"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
+	"github.com/tektoncd/pipeline/test/parse"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	knativetest "knative.dev/pkg/test"
@@ -45,6 +44,11 @@ const (
 
 // TestTaskRun is an integration test that will verify a TaskRun using kaniko
 func TestKanikoTaskRun(t *testing.T) {
+	var (
+		namespace, repo string
+		c               *clients
+	)
+
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -53,11 +57,23 @@ func TestKanikoTaskRun(t *testing.T) {
 		t.Skip("Skip test as skipRootUserTests set to true")
 	}
 
-	c, namespace := setup(ctx, t, withRegistry)
+	repo, err := getDockerRepo("kanikotasktest")
+	if err != nil {
+		// No KO_DOCKER_REPO set, use a sidecar registry instead
+		c, namespace = setup(ctx, t, withRegistry)
+		repo = fmt.Sprintf("registry.%s.svc.cluster.local:5000/kanikotasktest", namespace)
+	} else {
+		// When KO_DOCKER_REPO is set, we might need a secret attached to the service account
+		// If CreateGCPServiceAccountSecret no secret has been requested
+		// If CreateGCPServiceAccountSecret returns an error, a secret was requested, but failed
+		hasSecretConfig, err := CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret")
+		if err != nil {
+			t.Fatalf("Failed to create kaniko creds: %v", err)
+		}
+		c, namespace = setup(ctx, t)
+	}
 	t.Parallel()
 
-	repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace)
-
 	knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf)
 	defer tearDown(ctx, t, c, namespace)
 
@@ -72,7 +88,7 @@ func TestKanikoTaskRun(t *testing.T) {
 	}
 
 	t.Logf("Creating Task %s", kanikoTaskName)
-	if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace), metav1.CreateOptions{}); err != nil {
+	if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace, hasSecretConfig), metav1.CreateOptions{}); err != nil {
 		t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err)
 	}
 
@@ -158,8 +174,8 @@ spec:
 `, kanikoImageResourceName, repo))
 }
 
-func getTask(t *testing.T, repo, namespace string) *v1beta1.Task {
-	return parse.MustParseTask(t, fmt.Sprintf(`
+func getTask(t *testing.T, repo, namespace string, withSecretConfig bool) *v1beta1.Task {
+	task := parse.MustParseTask(t, fmt.Sprintf(`
 metadata:
   name: %s
   namespace: %s
@@ -177,16 +193,36 @@ spec:
     args: ['--dockerfile=/workspace/gitsource/integration/dockerfiles/Dockerfile_test_label',
            '--destination=%s',
 		   '--context=/workspace/gitsource',
-		   '--oci-layout-path=/workspace/output/builtImage',
-		   '--insecure',
-		   '--insecure-pull',
-		   '--insecure-registry=registry.%s:5000/']
+		   '--oci-layout-path=/workspace/output/builtImage']
     securityContext:
       runAsUser: 0
   sidecars:
   - name: registry
     image: %s
-`, kanikoTaskName, namespace, getTestImage(kanikoImage), repo, namespace, getTestImage(registryImage)))
+`, kanikoTaskName, namespace, getTestImage(kanikoImage), repo, getTestImage(registryImage)))
+	if withSecretConfig {
+		// Mount an extra volume to the kaniko step and set the GOOGLE_APPLICATION_CREDENTIALS env
+		task.Volumes = []corev1.Volume{{
+			corev1.SecretVolumeSource{
+				SecretName: "kaniko-secret",
+			}
+		}}
+		task.Spec.Steps[0].Env = append(
+			task.Spec.Steps[0].Env,
+			corev1.EnvVar{
+				Name: "GOOGLE_APPLICATION_CREDENTIALS",
+				Value: "/secrets/config.json",
+			},
+		)
+		task.Spec.Steps[0].VolumeMounts = append(
+			task.Spec.Steps[0].VolumeMounts,
+			corev1.VolumeMount{
+				Name: "kaniko-secret",
+				MountPath: "/secrets",
+			}
+		)
+	}
+	return task
 }
 
 func getTaskRun(t *testing.T, namespace string) *v1beta1.TaskRun {

diff --git a/test/ko_test.go b/test/ko_test.go
@@ -30,24 +30,24 @@ var (
 	missingKoFatal = "true"
 )
 
-func ensureDockerRepo(t *testing.T) string {
-	repo, err := getDockerRepo()
+func ensureDockerRepo(t *testing.T, name string) string {
+	repo, err := getDockerRepo(name)
 	if err != nil {
 		if missingKoFatal == "false" {
-			t.Skip("KO_DOCKER_REPO env variable is required")
+			t.Skip("KO_DOCKER_REPO env variable is required for %s", name)
 		}
-		t.Fatal("KO_DOCKER_REPO env variable is required")
+		t.Fatal("KO_DOCKER_REPO env variable is required for %s", name)
 	}
 	return repo
 }
 
-func getDockerRepo() (string, error) {
+func getDockerRepo(name string) (string, error) {
 	// according to knative/test-infra readme (https://github.com/knative/test-infra/blob/13055d769cc5e1756e605fcb3bcc1c25376699f1/scripts/README.md)
 	// the KO_DOCKER_REPO will be set with according to the project where the cluster is created
 	// it is used here to dynamically get the docker registry to push the image to
 	dockerRepo := os.Getenv("KO_DOCKER_REPO")
 	if dockerRepo == "" {
-		return "", errors.New("KO_DOCKER_REPO env variable is required")
+		return "", errors.New("KO_DOCKER_REPO env variable is not set")
 	}
-	return fmt.Sprintf("%s/kanikotasktest", dockerRepo), nil
+	return fmt.Sprintf("%s/%s", dockerRepo, name), nil
 }
diff --git a/test/v1alpha1/kaniko_task_test.go b/test/v1alpha1/kaniko_task_test.go
@@ -43,17 +43,35 @@ const (
 
 // TestTaskRun is an integration test that will verify a TaskRun using kaniko
 func TestKanikoTaskRun(t *testing.T) {
-	if skipRootUserTests {
-		t.Skip("Skip test as skipRootUserTests set to true")
-	}
+	var (
+		namespace, repo string
+		c               *clients
+	)
 
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	c, namespace := setup(ctx, t, withRegistry)
-	t.Parallel()
 
-	repo := fmt.Sprintf("registry.%s:5000/kanikotasktest", namespace)
+	if skipRootUserTests {
+		t.Skip("Skip test as skipRootUserTests set to true")
+	}
+
+	repo, err := getDockerRepo("kanikotasktest")
+	if err != nil {
+		// No KO_DOCKER_REPO set, use a sidecar registry instead
+		c, namespace = setup(ctx, t, withRegistry)
+		repo = fmt.Sprintf("registry.%s.svc.cluster.local:5000/kanikotasktest", namespace)
+	} else {
+		// When KO_DOCKER_REPO is set, we might need a secret attached to the service account
+		// If CreateGCPServiceAccountSecret no secret has been requested
+		// If CreateGCPServiceAccountSecret returns an error, a secret was requested, but failed
+		hasSecretConfig, err := CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, "kaniko-secret")
+		if err != nil {
+			t.Fatalf("Failed to create kaniko creds: %v", err)
+		}
+		c, namespace = setup(ctx, t)
+	}
+	t.Parallel()
 
 	knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf)
 	defer tearDown(ctx, t, c, namespace)
@@ -69,7 +87,7 @@ func TestKanikoTaskRun(t *testing.T) {
 	}
 
 	t.Logf("Creating Task %s", kanikoTaskName)
-	if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace), metav1.CreateOptions{}); err != nil {
+	if _, err := c.TaskClient.Create(ctx, getTask(t, repo, namespace, hasSecretConfig), metav1.CreateOptions{}); err != nil {
 		t.Fatalf("Failed to create Task `%s`: %s", kanikoTaskName, err)
 	}
 
@@ -152,8 +170,8 @@ spec:
 `, kanikoImageResourceName, repo))
 }
 
-func getTask(t *testing.T, repo, namespace string) *v1alpha1.Task {
-	return parse.MustParseAlphaTask(t, fmt.Sprintf(`
+func getTask(t *testing.T, repo, namespace string, withSecretConfig bool) *v1alpha1.Task {
+	task :=  parse.MustParseAlphaTask(t, fmt.Sprintf(`
 metadata:
   name: %s
 spec:
@@ -171,16 +189,36 @@ spec:
     args: ['--dockerfile=/workspace/gitsource/integration/dockerfiles/Dockerfile_test_label',
            '--destination=%s',
 		   '--context=/workspace/gitsource',
-		   '--oci-layout-path=/workspace/output/builtImage',
-		   '--insecure',
-		   '--insecure-pull',
-		   '--insecure-registry=registry.%s:5000/']
+		   '--oci-layout-path=/workspace/output/builtImage']
     securityContext:
       runAsUser: 0
   sidecars:
   - name: registry
     image: registry
-`, kanikoTaskName, repo, namespace))
+`, kanikoTaskName, repo))
+	if withSecretConfig {
+		// Mount an extra volume to the kaniko step and set the GOOGLE_APPLICATION_CREDENTIALS env
+		task.Volumes = []corev1.Volume{{
+			corev1.SecretVolumeSource{
+				SecretName: "kaniko-secret",
+			}
+		}}
+		task.Spec.Steps[0].Env = append(
+			task.Spec.Steps[0].Env,
+			corev1.EnvVar{
+				Name: "GOOGLE_APPLICATION_CREDENTIALS",
+				Value: "/secrets/config.json",
+			},
+		)
+		task.Spec.Steps[0].VolumeMounts = append(
+			task.Spec.Steps[0].VolumeMounts,
+			corev1.VolumeMount{
+				Name: "kaniko-secret",
+				MountPath: "/secrets",
+			}
+		)
+	}
+	return task
 }
 
 func getTaskRun(t *testing.T, namespace string) *v1alpha1.TaskRun {