Update build-gcs resource type to support .tar.gz archives

Gopkg.toml

@@ -3,7 +3,6 @@
 
 required = [
   "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-fetcher",
-  "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader",
   "github.com/knative/test-infra/tools/dep-collector",
   "github.com/tektoncd/plumbing/scripts",
   "k8s.io/apimachinery/pkg/util/sets/types",

config/controller.yaml

@@ -42,7 +42,6 @@ spec:
           "-imagedigest-exporter-image", "github.com/tektoncd/pipeline/cmd/imagedigestexporter",
           "-pr-image", "github.com/tektoncd/pipeline/cmd/pullrequest-init",
           "-build-gcs-fetcher-image", "github.com/tektoncd/pipeline/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-fetcher",
-          "-build-gcs-uploader-image", "github.com/tektoncd/pipeline/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader",
         ]
         volumeMounts:
         - name: config-logging

docs/resources.md

@@ -590,18 +590,15 @@ Params that can be added are the following:
 1.  `artifactType`: represent the type of GCS resource. Right now, we support
     following types:
 
-    -   `Archive`:
-        -   Archive indicates that resource fetched is an archive file.
-            Currently, Build GCS resource only supports `.zip` archive.
-        -   It unzips the archive and places all the files in the directory
-            which is set at runtime.
-        -   If `artifactType` is set to `Archive`, `location` should point to a
-            `.zip` file.
-    -   [`Manifest`](https://github.com/GoogleCloudPlatform/cloud-builders/tree/master/gcs-fetcher#source-manifests):
-        -   Manifest indicates that resource should be fetched using a source
-            manifest file.
-        -   If `artifactType` is set to `Manifest`, `location` should point to a
-            source manifest file.
+* `ZipArchive`:
+  * ZipArchive indicates that resource fetched is an archive file in the zip format.
+  * It unzips the archive and places all the files in the directory which is set at runtime.
+  * `Archive` is also supported and is equivalent to `ZipArchive`, but is deprecated.
+* `TarGzArchive`:
+  * TarGzArchive indicates that resource fetched is a gzipped archive file in the tar format.
+  * It unzips the archive and places all the files in the directory which is set at runtime.
+* [`Manifest`](https://github.com/GoogleCloudPlatform/cloud-builders/tree/master/gcs-fetcher#source-manifests):
+  * Manifest indicates that resource should be fetched using a source manifest file.
 
 Private buckets other than ones accessible by
 [TaskRun Service Account](./taskruns.md#service-account) can not be configured

examples/taskruns/gcs-resource-spec-taskrun.yaml → examples/taskruns/build-gcs-targz.yaml

@@ -1,27 +1,25 @@
 apiVersion: tekton.dev/v1alpha1
 kind: TaskRun
 metadata:
-  name: list-file
+  name: build-gcs-targz
 spec:
   taskSpec:
     inputs:
       resources:
-      - name: rules
+      - name: source
         type: storage
     steps:
-    - name: list
-      image: ubuntu
-      command: ["/bin/bash"]
-      args: ['-c', 'ls -al /workspace/rules/rules_docker-master'] # tests build-gcs resource
+    - image: ubuntu
+      command: ['cat', 'source/file.txt']  # tests build-gcs resource
   inputs:
     resources:
-    - name: rules
+    - name: source
       resourceSpec:
         type: storage
         params:
           - name: location
-            value: gs://build-crd-tests/rules_docker-master.zip
+            value: gs://build-crd-tests/archive.tar.gz
           - name: artifactType
-            value: Archive
+            value: TarGzArchive
           - name: type
             value: build-gcs

examples/taskruns/build-gcs-zip.yaml

@@ -0,0 +1,25 @@
+apiVersion: tekton.dev/v1alpha1
+kind: TaskRun
+metadata:
+  name: build-gcs-zip
+spec:
+  taskSpec:
+    inputs:
+      resources:
+      - name: source
+        type: storage
+    steps:
+    - image: ubuntu
+      command: ['cat', 'source/file.txt']  # tests build-gcs resource
+  inputs:
+    resources:
+    - name: source
+      resourceSpec:
+        type: storage
+        params:
+          - name: location
+            value: gs://build-crd-tests/archive.zip
+          - name: artifactType
+            value: ZipArchive
+          - name: type
+            value: build-gcs

pkg/apis/pipeline/v1alpha1/build_gcs_resource.go

@@ -29,25 +29,38 @@ import (
 var (
 	buildGCSFetcherImage = flag.String("build-gcs-fetcher-image", "gcr.io/cloud-builders/gcs-fetcher:latest",
 		"The container image containing our GCS fetcher binary.")
-	buildGCSUploaderImage = flag.String("build-gcs-uploader-image", "gcr.io/cloud-builders/gcs-uploader:latest",
-		"The container image containing our GCS uploader binary.")
 )
 
 // GCSArtifactType defines a type of GCS resource.
 type GCSArtifactType string
 
 const (
-	// GCSArchive indicates that resource should be fetched from a typical archive file.
+	// GCSZipArchive indicates that the resource should be fetched and
+	// extracted as a .zip file.
+	//
+	// Deprecated: Use GCSZipArchive instead.
 	GCSArchive GCSArtifactType = "Archive"
 
+	// GCSZipArchive indicates that the resource should be fetched and
+	// extracted as a .zip file.
+	GCSZipArchive GCSArtifactType = "ZipArchive"
+
+	// GCSTarGzArchive indicates that the resource should be fetched and
+	// extracted as a .tar.gz file.
+	GCSTarGzArchive GCSArtifactType = "TarGzArchive"
+
 	// GCSManifest indicates that resource should be fetched using a
 	// manifest-based protocol which enables incremental source upload.
 	GCSManifest GCSArtifactType = "Manifest"
-
-	// EmptyArtifactType indicates, no artifact type is specified.
-	EmptyArtifactType = ""
 )
 
+var validArtifactTypes = []GCSArtifactType{
+	GCSArchive,
+	GCSManifest,
+	GCSZipArchive,
+	GCSTarGzArchive,
+}
+
 // BuildGCSResource describes a resource in the form of an archive,
 // or a source manifest describing files to fetch.
 // BuildGCSResource does incremental uploads for files in  directory.
@@ -69,7 +82,6 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) {
 	}
 	var location string
 	var aType GCSArtifactType
-
 	for _, param := range r.Spec.Params {
 		switch {
 		case strings.EqualFold(param.Name, "Location"):
@@ -85,8 +97,8 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) {
 	if location == "" {
 		return nil, xerrors.Errorf("BuildGCSResource: Need Location to be specified in order to create BuildGCS resource %s", r.Name)
 	}
-	if aType == EmptyArtifactType {
-		return nil, xerrors.Errorf("BuildGCSResource: Need ArtifactType to be specified in order to fetch BuildGCS resource %s", r.Name)
+	if aType == GCSArtifactType("") {
+		return nil, xerrors.Errorf("BuildGCSResource: Need ArtifactType to be specified to create BuildGCS resource %s", r.Name)
 	}
 	return &BuildGCSResource{
 		Name:         r.Name,
@@ -96,20 +108,24 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) {
 	}, nil
 }
 
-// GetName returns the name of the resource
-func (s BuildGCSResource) GetName() string {
-	return s.Name
-}
+// GetName returns the name of the resource.
+func (s BuildGCSResource) GetName() string { return s.Name }
 
-// GetType returns the type of the resource, in this case "storage"
-func (s BuildGCSResource) GetType() PipelineResourceType {
-	return PipelineResourceTypeStorage
-}
+// GetType returns the type of the resource, in this case "storage".
+func (s BuildGCSResource) GetType() PipelineResourceType { return PipelineResourceTypeStorage }
 
-// GetSecretParams returns the resource secret params
+// GetSecretParams returns nil because it takes no secret params.
 func (s *BuildGCSResource) GetSecretParams() []SecretParam { return nil }
 
-// Replacements is used for template replacement on an GCSResource inside of a Taskrun.
+// GetUploadSteps returns nil because it does not support uploading as an
+// output resource.
+func (s *BuildGCSResource) GetUploadSteps(string) ([]Step, error) { return nil, nil }
+
+// GetUploadVolumeSpec returns nil because it does not support uploading as an
+// output resource.
+func (s *BuildGCSResource) GetUploadVolumeSpec(*TaskSpec) ([]corev1.Volume, error) { return nil, nil }
+
+// Replacements returns the set of available replacements for this resource.
 func (s *BuildGCSResource) Replacements() map[string]string {
 	return map[string]string{
 		"name":     s.Name,
@@ -118,7 +134,8 @@ func (s *BuildGCSResource) Replacements() map[string]string {
 	}
 }
 
-// GetDownloadSteps returns an array of container specs to download gcs storage object
+// GetDownloadSteps returns the Steps needed to populate the workspace with the
+// resource's data.
 func (s *BuildGCSResource) GetDownloadSteps(sourcePath string) ([]Step, error) {
 	args := []string{"--type", string(s.ArtifactType), "--location", s.Location}
 	// dest_dir is the destination directory for GCS files to be copies"
@@ -129,48 +146,24 @@ func (s *BuildGCSResource) GetDownloadSteps(sourcePath string) ([]Step, error) {
 	return []Step{
 		CreateDirStep(s.Name, sourcePath),
 		{Container: corev1.Container{
-			Name:  names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("storage-fetch-%s", s.Name)),
-			Image: *buildGCSFetcherImage,
-			Args:  args,
+			Name:    names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("storage-fetch-%s", s.Name)),
+			Command: []string{"/ko-app/gcs-fetcher"},
+			Image:   *buildGCSFetcherImage,
+			Args:    args,
 		}}}, nil
 }
 
-// GetUploadSteps gets container spec for gcs resource to be uploaded like
-// set environment variable from secret params and set volume mounts for those secrets
-func (s *BuildGCSResource) GetUploadSteps(sourcePath string) ([]Step, error) {
-	if s.ArtifactType != GCSManifest {
-		return nil, xerrors.Errorf("BuildGCSResource: Can only upload Artifacts of type Manifest: %s", s.Name)
-	}
-	if sourcePath == "" {
-		return nil, xerrors.Errorf("BuildGCSResource: Expect Destination Directory param to be set %s", s.Name)
-	}
-	args := []string{"--location", s.Location, "--dir", sourcePath}
-
-	return []Step{{Container: corev1.Container{
-		Name:  names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("storage-upload-%s", s.Name)),
-		Image: *buildGCSUploaderImage,
-		Args:  args,
-	}}}, nil
-}
-
 func getArtifactType(val string) (GCSArtifactType, error) {
-	aType := GCSArtifactType(val)
-	valid := []string{string(GCSArchive), string(GCSManifest)}
-	switch aType {
-	case GCSArchive:
-		return aType, nil
-	case GCSManifest:
-		return aType, nil
-	case EmptyArtifactType:
-		return "", xerrors.Errorf("ArtifactType is empty. Should be one of %s", strings.Join(valid, ","))
+	a := GCSArtifactType(val)
+	for _, v := range validArtifactTypes {
+		if a == v {
+			return a, nil
+		}
 	}
-	return "", xerrors.Errorf("Invalid ArtifactType %s. Should be one of %s", aType, strings.Join(valid, ","))
-}
-
-func (s *BuildGCSResource) GetUploadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) {
-	return getStorageUploadVolumeSpec(s, spec)
+	return "", xerrors.Errorf("Invalid ArtifactType %s. Should be one of %s", val, validArtifactTypes)
 }
 
+// GetDownloadVolumeSpec returns the volumes needed by this resource.
 func (s *BuildGCSResource) GetDownloadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) {
-	return getStorageUploadVolumeSpec(s, spec)
+	return getStorageVolumeSpec(s, spec)
 }