diff --git a/Gopkg.lock b/Gopkg.lock index df738c46b3d..afac4fcc9a1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -68,14 +68,12 @@ version = "v11.1.2" [[projects]] - digest = "1:114566651114ab2375768593255e8b0590e36e603c3903399c20a590bcf9af85" + digest = "1:81ed671dd39c40361f18453874283bf8a4824cbed76387dff4954005a221e40e" name = "github.com/GoogleCloudPlatform/cloud-builders" packages = [ "gcs-fetcher/cmd/gcs-fetcher", - "gcs-fetcher/cmd/gcs-uploader", "gcs-fetcher/pkg/common", "gcs-fetcher/pkg/fetcher", - "gcs-fetcher/pkg/uploader", ] pruneopts = "NUT" revision = "073e4daa3d0dcb50173fdac5581709a17b97e9ed" @@ -1311,7 +1309,6 @@ analyzer-version = 1 input-imports = [ "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-fetcher", - "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader", "github.com/cloudevents/sdk-go/pkg/cloudevents", "github.com/cloudevents/sdk-go/pkg/cloudevents/client", "github.com/cloudevents/sdk-go/pkg/cloudevents/context", diff --git a/Gopkg.toml b/Gopkg.toml index 494c46646ef..399a26835dc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -3,7 +3,6 @@ required = [ "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-fetcher", - "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader", "github.com/knative/test-infra/tools/dep-collector", "github.com/tektoncd/plumbing/scripts", "k8s.io/apimachinery/pkg/util/sets/types", diff --git a/config/controller.yaml b/config/controller.yaml index 579398b89d0..ff25d78917b 100644 --- a/config/controller.yaml +++ b/config/controller.yaml @@ -42,7 +42,6 @@ spec: "-imagedigest-exporter-image", "github.com/tektoncd/pipeline/cmd/imagedigestexporter", "-pr-image", "github.com/tektoncd/pipeline/cmd/pullrequest-init", "-build-gcs-fetcher-image", "github.com/tektoncd/pipeline/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-fetcher", - "-build-gcs-uploader-image", "github.com/tektoncd/pipeline/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader", ] volumeMounts: - name: config-logging diff --git a/docs/resources.md b/docs/resources.md index 87aeefb83a5..f10040ac9ec 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -590,18 +590,21 @@ Params that can be added are the following: 1. `artifactType`: represent the type of GCS resource. Right now, we support following types: - - `Archive`: - - Archive indicates that resource fetched is an archive file. - Currently, Build GCS resource only supports `.zip` archive. - - It unzips the archive and places all the files in the directory - which is set at runtime. - - If `artifactType` is set to `Archive`, `location` should point to a - `.zip` file. + - `ZipArchive`: + - ZipArchive indicates that resource fetched is an archive file in + the zip format. + - It unzips the archive and places all the files in the directory + which is set at runtime. + - `Archive` is also supported and is equivalent to `ZipArchive`, but + is deprecated. + - `TarGzArchive`: + - TarGzArchive indicates that resource fetched is a gzipped archive file + in the tar format. + - It unzips the archive and places all the files in the directory + which is set at runtime. - [`Manifest`](https://github.com/GoogleCloudPlatform/cloud-builders/tree/master/gcs-fetcher#source-manifests): - Manifest indicates that resource should be fetched using a source manifest file. - - If `artifactType` is set to `Manifest`, `location` should point to a - source manifest file. Private buckets other than ones accessible by [TaskRun Service Account](./taskruns.md#service-account) can not be configured diff --git a/examples/taskruns/gcs-resource-spec-taskrun.yaml b/examples/taskruns/build-gcs-targz.yaml similarity index 51% rename from examples/taskruns/gcs-resource-spec-taskrun.yaml rename to examples/taskruns/build-gcs-targz.yaml index abdc17b0e60..d6ae341eda7 100644 --- a/examples/taskruns/gcs-resource-spec-taskrun.yaml +++ b/examples/taskruns/build-gcs-targz.yaml @@ -1,27 +1,25 @@ apiVersion: tekton.dev/v1alpha1 kind: TaskRun metadata: - name: list-file + name: build-gcs-targz spec: taskSpec: inputs: resources: - - name: rules + - name: source type: storage steps: - - name: list - image: ubuntu - command: ["/bin/bash"] - args: ['-c', 'ls -al /workspace/rules/rules_docker-master'] # tests build-gcs resource + - image: ubuntu + command: ['cat', 'source/file.txt'] # tests build-gcs resource inputs: resources: - - name: rules + - name: source resourceSpec: type: storage params: - name: location - value: gs://build-crd-tests/rules_docker-master.zip + value: gs://build-crd-tests/archive.tar.gz - name: artifactType - value: Archive + value: TarGzArchive - name: type value: build-gcs diff --git a/examples/taskruns/build-gcs-zip.yaml b/examples/taskruns/build-gcs-zip.yaml new file mode 100644 index 00000000000..cce02ccf7ea --- /dev/null +++ b/examples/taskruns/build-gcs-zip.yaml @@ -0,0 +1,25 @@ +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + name: build-gcs-zip +spec: + taskSpec: + inputs: + resources: + - name: source + type: storage + steps: + - image: ubuntu + command: ['cat', 'source/file.txt'] # tests build-gcs resource + inputs: + resources: + - name: source + resourceSpec: + type: storage + params: + - name: location + value: gs://build-crd-tests/archive.zip + - name: artifactType + value: ZipArchive + - name: type + value: build-gcs diff --git a/pkg/apis/pipeline/v1alpha1/build_gcs_resource.go b/pkg/apis/pipeline/v1alpha1/build_gcs_resource.go index 99dcbda2402..28d010b657f 100644 --- a/pkg/apis/pipeline/v1alpha1/build_gcs_resource.go +++ b/pkg/apis/pipeline/v1alpha1/build_gcs_resource.go @@ -29,8 +29,6 @@ import ( var ( buildGCSFetcherImage = flag.String("build-gcs-fetcher-image", "gcr.io/cloud-builders/gcs-fetcher:latest", "The container image containing our GCS fetcher binary.") - buildGCSUploaderImage = flag.String("build-gcs-uploader-image", "gcr.io/cloud-builders/gcs-uploader:latest", - "The container image containing our GCS uploader binary.") ) // GCSArtifactType defines a type of GCS resource. @@ -38,16 +36,27 @@ type GCSArtifactType string const ( // GCSArchive indicates that resource should be fetched from a typical archive file. + // + // Deprecated: Use GCSZipArchive instead. GCSArchive GCSArtifactType = "Archive" + // TODO: docs + GCSZipArchive GCSArtifactType = "ZipArchive" + + GCSTarGzArchive GCSArtifactType = "TarGzArchive" + // GCSManifest indicates that resource should be fetched using a // manifest-based protocol which enables incremental source upload. GCSManifest GCSArtifactType = "Manifest" - - // EmptyArtifactType indicates, no artifact type is specified. - EmptyArtifactType = "" ) +var validArtifactTypes = []GCSArtifactType{ + GCSArchive, + GCSManifest, + GCSZipArchive, + GCSTarGzArchive, +} + // BuildGCSResource describes a resource in the form of an archive, // or a source manifest describing files to fetch. // BuildGCSResource does incremental uploads for files in directory. @@ -69,7 +78,6 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) { } var location string var aType GCSArtifactType - for _, param := range r.Spec.Params { switch { case strings.EqualFold(param.Name, "Location"): @@ -85,8 +93,8 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) { if location == "" { return nil, xerrors.Errorf("BuildGCSResource: Need Location to be specified in order to create BuildGCS resource %s", r.Name) } - if aType == EmptyArtifactType { - return nil, xerrors.Errorf("BuildGCSResource: Need ArtifactType to be specified in order to fetch BuildGCS resource %s", r.Name) + if aType == GCSArtifactType("") { + return nil, xerrors.Errorf("BuildGCSResource: Neet ArtifactType to be specified to create BuildGCS resource %s", r.Name) } return &BuildGCSResource{ Name: r.Name, @@ -96,18 +104,11 @@ func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) { }, nil } -// GetName returns the name of the resource -func (s BuildGCSResource) GetName() string { - return s.Name -} - -// GetType returns the type of the resource, in this case "storage" -func (s BuildGCSResource) GetType() PipelineResourceType { - return PipelineResourceTypeStorage -} - -// GetSecretParams returns the resource secret params -func (s *BuildGCSResource) GetSecretParams() []SecretParam { return nil } +func (s BuildGCSResource) GetName() string { return s.Name } +func (s BuildGCSResource) GetType() PipelineResourceType { return PipelineResourceTypeStorage } +func (s *BuildGCSResource) GetSecretParams() []SecretParam { return nil } +func (s *BuildGCSResource) GetUploadSteps(string) ([]Step, error) { return nil, nil } +func (s *BuildGCSResource) GetUploadVolumeSpec(*TaskSpec) ([]corev1.Volume, error) { return nil, nil } // Replacements is used for template replacement on an GCSResource inside of a Taskrun. func (s *BuildGCSResource) Replacements() map[string]string { @@ -135,42 +136,16 @@ func (s *BuildGCSResource) GetDownloadSteps(sourcePath string) ([]Step, error) { }}}, nil } -// GetUploadSteps gets container spec for gcs resource to be uploaded like -// set environment variable from secret params and set volume mounts for those secrets -func (s *BuildGCSResource) GetUploadSteps(sourcePath string) ([]Step, error) { - if s.ArtifactType != GCSManifest { - return nil, xerrors.Errorf("BuildGCSResource: Can only upload Artifacts of type Manifest: %s", s.Name) - } - if sourcePath == "" { - return nil, xerrors.Errorf("BuildGCSResource: Expect Destination Directory param to be set %s", s.Name) - } - args := []string{"--location", s.Location, "--dir", sourcePath} - - return []Step{{Container: corev1.Container{ - Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("storage-upload-%s", s.Name)), - Image: *buildGCSUploaderImage, - Args: args, - }}}, nil -} - func getArtifactType(val string) (GCSArtifactType, error) { - aType := GCSArtifactType(val) - valid := []string{string(GCSArchive), string(GCSManifest)} - switch aType { - case GCSArchive: - return aType, nil - case GCSManifest: - return aType, nil - case EmptyArtifactType: - return "", xerrors.Errorf("ArtifactType is empty. Should be one of %s", strings.Join(valid, ",")) + a := GCSArtifactType(val) + for _, v := range validArtifactTypes { + if a == v { + return a, nil + } } - return "", xerrors.Errorf("Invalid ArtifactType %s. Should be one of %s", aType, strings.Join(valid, ",")) -} - -func (s *BuildGCSResource) GetUploadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) { - return getStorageUploadVolumeSpec(s, spec) + return "", xerrors.Errorf("Invalid ArtifactType %s. Should be one of %s", val, validArtifactTypes) } func (s *BuildGCSResource) GetDownloadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) { - return getStorageUploadVolumeSpec(s, spec) + return getStorageVolumeSpec(s, spec) } diff --git a/pkg/apis/pipeline/v1alpha1/build_gcs_resource_test.go b/pkg/apis/pipeline/v1alpha1/build_gcs_resource_test.go index 6d75e9a37eb..1892efbfdff 100644 --- a/pkg/apis/pipeline/v1alpha1/build_gcs_resource_test.go +++ b/pkg/apis/pipeline/v1alpha1/build_gcs_resource_test.go @@ -126,71 +126,49 @@ func Test_BuildGCSGetReplacements(t *testing.T) { } func Test_BuildGCSGetDownloadSteps(t *testing.T) { - resource := &v1alpha1.BuildGCSResource{ - Name: "gcs-valid", - Location: "gs://some-bucket", - ArtifactType: "Archive", - } - wantSteps := []v1alpha1.Step{{Container: corev1.Container{ - Name: "create-dir-gcs-valid-9l9zj", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace"}, - }}, {Container: corev1.Container{ - Name: "storage-fetch-gcs-valid-mz4c7", - Image: "gcr.io/cloud-builders/gcs-fetcher:latest", - Args: []string{"--type", "Archive", "--location", "gs://some-bucket", - "--dest_dir", "/workspace"}, - }}} - names.TestingSeed() - got, err := resource.GetDownloadSteps("/workspace") - if err != nil { - t.Fatalf("GetDownloadSteps: %v", err) - } - if d := cmp.Diff(got, wantSteps); d != "" { - t.Errorf("Error mismatch between download steps: %s", d) - } -} - -func Test_BuildGCSGetUploadSteps(t *testing.T) { - for _, tc := range []struct { - name string - resource *v1alpha1.BuildGCSResource - wantSteps []v1alpha1.Step - wantErr bool - }{{ - name: "valid upload to protected buckets with directory paths", - resource: &v1alpha1.BuildGCSResource{ - Name: "gcs-valid", - Location: "gs://some-bucket/manifest.json", - ArtifactType: "Manifest", - }, - wantSteps: []v1alpha1.Step{{Container: corev1.Container{ - Name: "storage-upload-gcs-valid-mssqb", - Image: "gcr.io/cloud-builders/gcs-uploader:latest", - Args: []string{"--location", "gs://some-bucket/manifest.json", "--dir", "/workspace"}, - }}}, - }, { - name: "invalid upload to protected buckets with single file", - resource: &v1alpha1.BuildGCSResource{ - Name: "gcs-valid", - ArtifactType: "Archive", - Location: "gs://some-bucket", - }, - wantErr: true, - }} { - t.Run(tc.name, func(t *testing.T) { - got, err := tc.resource.GetUploadSteps("/workspace") - if tc.wantErr && err == nil { - t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err) + for _, at := range []v1alpha1.GCSArtifactType{ + v1alpha1.GCSArchive, + v1alpha1.GCSZipArchive, + v1alpha1.GCSTarGzArchive, + v1alpha1.GCSManifest, + } { + t.Run(string(at), func(t *testing.T) { + resource := &v1alpha1.BuildGCSResource{ + Name: "gcs-valid", + Location: "gs://some-bucket", + ArtifactType: at, } - if !tc.wantErr && err != nil { - t.Fatalf("GetUploadSteps: %v", err) + wantSteps := []v1alpha1.Step{{Container: corev1.Container{ + Name: "create-dir-gcs-valid-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /workspace"}, + }}, {Container: corev1.Container{ + Name: "storage-fetch-gcs-valid-mz4c7", + Image: "gcr.io/cloud-builders/gcs-fetcher:latest", + Args: []string{"--type", string(at), "--location", "gs://some-bucket", + "--dest_dir", "/workspace"}, + }}} + names.TestingSeed() + got, err := resource.GetDownloadSteps("/workspace") + if err != nil { + t.Fatalf("GetDownloadSteps: %v", err) } - - if d := cmp.Diff(got, tc.wantSteps); d != "" { - t.Errorf("Error mismatch between upload steps: %s", d) + if d := cmp.Diff(got, wantSteps); d != "" { + t.Errorf("Error mismatch between download steps: %s", d) } }) } } + +func TestBuildGCS_InvalidArtifactType(t *testing.T) { + pr := tb.PipelineResource("build-gcs-resource", "default", tb.PipelineResourceSpec( + v1alpha1.PipelineResourceTypeStorage, + tb.PipelineResourceSpecParam("Location", "gs://fake-bucket"), + tb.PipelineResourceSpecParam("type", "build-gcs"), + tb.PipelineResourceSpecParam("ArtifactType", "InVaLiD"), + )) + if _, err := v1alpha1.NewBuildGCSResource(pr); err == nil { + t.Error("NewBuildGCSResource: expected error") + } +} diff --git a/pkg/apis/pipeline/v1alpha1/gcs_resource.go b/pkg/apis/pipeline/v1alpha1/gcs_resource.go index 24064273a6d..157b8ff0970 100644 --- a/pkg/apis/pipeline/v1alpha1/gcs_resource.go +++ b/pkg/apis/pipeline/v1alpha1/gcs_resource.go @@ -145,9 +145,9 @@ func (s *GCSResource) GetDownloadSteps(sourcePath string) ([]Step, error) { } func (s *GCSResource) GetUploadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) { - return getStorageUploadVolumeSpec(s, spec) + return getStorageVolumeSpec(s, spec) } func (s *GCSResource) GetDownloadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) { - return getStorageUploadVolumeSpec(s, spec) + return getStorageVolumeSpec(s, spec) } diff --git a/pkg/apis/pipeline/v1alpha1/storage_resource.go b/pkg/apis/pipeline/v1alpha1/storage_resource.go index fbe271fbf97..58e46b91b33 100644 --- a/pkg/apis/pipeline/v1alpha1/storage_resource.go +++ b/pkg/apis/pipeline/v1alpha1/storage_resource.go @@ -58,7 +58,7 @@ func NewStorageResource(r *PipelineResource) (PipelineStorageResourceInterface, return nil, xerrors.Errorf("StoreResource: Cannot create a storage resource without type %s in spec", r.Name) } -func getStorageUploadVolumeSpec(s PipelineStorageResourceInterface, spec *TaskSpec) ([]corev1.Volume, error) { +func getStorageVolumeSpec(s PipelineStorageResourceInterface, spec *TaskSpec) ([]corev1.Volume, error) { var storageVol []corev1.Volume mountedSecrets := map[string]string{} diff --git a/tekton/publish-run.yaml b/tekton/publish-run.yaml index 671be2a7876..1fe67d73c9e 100644 --- a/tekton/publish-run.yaml +++ b/tekton/publish-run.yaml @@ -231,6 +231,3 @@ spec: - name: builtGcsFetcherImage resourceRef: name: gcs-fetcher-image - - name: builtGcsUploaderImage - resourceRef: - name: gcs-uploader-image \ No newline at end of file diff --git a/tekton/publish.yaml b/tekton/publish.yaml index 0d689cd76a7..87765ccabbc 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -47,8 +47,6 @@ spec: type: image - name: builtGcsFetcherImage type: image - - name: builtGcsUploaderImage - type: image steps: - name: build-push-base-images @@ -165,7 +163,6 @@ spec: $(inputs.params.imageRegistry)/$(inputs.params.pathToProject)/$(outputs.resources.builtDigestExporterImage.url) $(inputs.params.imageRegistry)/$(inputs.params.pathToProject)/$(outputs.resources.builtPullRequestInitImage.url) $(inputs.params.imageRegistry)/$(inputs.params.pathToProject)/$(outputs.resources.builtGcsFetcherImage.url) - $(inputs.params.imageRegistry)/$(inputs.params.pathToProject)/$(outputs.resources.builtGcsUploaderImage.url) ) # Parse the built images from the release.yaml generated by ko BUILT_IMAGES=( $(/workspace/go/src/github.com/tektoncd/pipeline/tekton/koparse/koparse.py --path /workspace/bucket/latest/release.yaml --base $(inputs.params.imageRegistry)/$(inputs.params.pathToProject) --images ${IMAGES[@]}) ) @@ -194,4 +191,4 @@ spec: volumes: - name: gcp-secret secret: - secretName: release-secret \ No newline at end of file + secretName: release-secret diff --git a/tekton/release-pipeline-run.yaml b/tekton/release-pipeline-run.yaml index fe740927add..61751ffeaed 100644 --- a/tekton/release-pipeline-run.yaml +++ b/tekton/release-pipeline-run.yaml @@ -60,6 +60,3 @@ spec: - name: builtGcsFetcherImage resourceRef: name: gcs-fetcher-image - - name: builtGcsUploaderImage - resourceRef: - name: gcs-uploader-image \ No newline at end of file diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 405eb5f0f33..12c4b45eded 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -43,8 +43,6 @@ spec: type: image - name: builtGcsFetcherImage type: image - - name: builtGcsUploaderImage - type: image tasks: - name: lint taskRef: @@ -126,5 +124,3 @@ spec: resource: builtPullRequestInitImage - name: builtGcsFetcherImage resource: builtGcsFetcherImage - - name: builtGcsUploaderImage - resource: builtGcsUploaderImage diff --git a/test/gcs_taskrun_test.go b/test/gcs_taskrun_test.go deleted file mode 100644 index 07fdd6f060c..00000000000 --- a/test/gcs_taskrun_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// +build e2e - -/* -Copyright 2019 The Tekton Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" - tb "github.com/tektoncd/pipeline/test/builder" - knativetest "knative.dev/pkg/test" -) - -// TestStorageTaskRun is an integration test that will verify GCS input resource runtime contract -// - adds volumes with expected name -// - places files in expected place - -func TestStorageTaskRun(t *testing.T) { - configFilePath := os.Getenv("GCP_SERVICE_ACCOUNT_KEY_PATH") - if configFilePath == "" { - t.Skip("GCP_SERVICE_ACCOUNT_KEY_PATH variable is not set.") - } - t.Parallel() - - c, namespace := setup(t) - knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf) - defer tearDown(t, c, namespace) - - secretName := "auth-secret" - _, err := CreateGCPServiceAccountSecret(t, c.KubeClient, namespace, secretName) - if err != nil { - t.Fatalf("could not create secret %s", err) - } - - resName := "gcs-resource" - if _, err := c.PipelineResourceClient.Create(getResources(namespace, resName, secretName, configFilePath)); err != nil { - t.Fatalf("Failed to create Pipeline Resource `%s`: %s", resName, err) - } - - taskRunName := "gcs-taskrun" - t.Logf("Creating Task and TaskRun %s in namespace %s", taskRunName, namespace) - - if _, err := c.TaskClient.Create(getGCSStorageTask(namespace)); err != nil { - t.Fatalf("Failed to create Task gcs-file : %s", err) - } - - if _, err := c.TaskRunClient.Create(getGCSTaskRun(namespace, taskRunName, resName)); err != nil { - t.Fatalf("Failed to create TaskRun %s: %s", taskRunName, err) - } - - t.Logf("Waiting for TaskRun %s in namespace %s to complete", taskRunName, namespace) - - if err := WaitForTaskRunState(c, taskRunName, TaskRunSucceed(taskRunName), "TaskRunSuccess"); err != nil { - t.Errorf("Error waiting for TaskRun %s to finish: %s", taskRunName, err) - } - t.Logf("TaskRun %s succeeded", taskRunName) -} - -func getGCSStorageTask(namespace string) *v1alpha1.Task { - return tb.Task("gcs-file", namespace, tb.TaskSpec( - tb.TaskInputs(tb.InputsResource("gcsbucket", v1alpha1.PipelineResourceTypeStorage, - tb.ResourceTargetPath("gcs-workspace"), - )), - tb.Step("read-gcs-bucket", "ubuntu", tb.StepCommand("/bin/bash"), - tb.StepArgs("-c", "ls -la /workspace/gcs-workspace/rules_docker-master.zip"), - ), - )) -} - -func getGCSTaskRun(namespace, name, resName string) *v1alpha1.TaskRun { - return tb.TaskRun(name, namespace, - tb.TaskRunSpec(tb.TaskRunTaskRef("gcs-file"), - tb.TaskRunInputs( - tb.TaskRunInputsResource("gcsbucket", tb.TaskResourceBindingRef(resName)), - ))) -} - -func getResources(namespace, name, secretName, configFile string) *v1alpha1.PipelineResource { - res := tb.PipelineResource(name, namespace, tb.PipelineResourceSpec( - v1alpha1.PipelineResourceTypeStorage, - tb.PipelineResourceSpecParam("location", "gs://build-crd-tests/rules_docker-master.zip"), - tb.PipelineResourceSpecParam("type", "gcs"), - )) - if configFile != "" { - jsonKeyFilename := filepath.Base(configFile) - tb.PipelineResourceSpecSecretParam("GOOGLE_APPLICATION_CREDENTIALS", secretName, jsonKeyFilename)(&res.Spec) - } - return res -} diff --git a/third_party/VENDOR-LICENSE b/third_party/VENDOR-LICENSE index c3ecece3f5c..1d329b5b59b 100644 --- a/third_party/VENDOR-LICENSE +++ b/third_party/VENDOR-LICENSE @@ -1469,6 +1469,213 @@ Import: github.com/tektoncd/pipeline/vendor/github.com/census-instrumentation/op +=========================================================== +Import: github.com/tektoncd/pipeline/vendor/github.com/cloudevents/sdk-go + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + =========================================================== Import: github.com/tektoncd/pipeline/vendor/github.com/davecgh/go-spew @@ -8782,6 +8989,213 @@ Import: github.com/tektoncd/pipeline/vendor/knative.dev/caching +=========================================================== +Import: github.com/tektoncd/pipeline/vendor/knative.dev/eventing-contrib + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + =========================================================== Import: github.com/tektoncd/pipeline/vendor/knative.dev/pkg diff --git a/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader/main.go b/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader/main.go deleted file mode 100644 index a5f050f88c1..00000000000 --- a/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/cmd/gcs-uploader/main.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2018 Google, Inc. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -import ( - "context" - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" - - "cloud.google.com/go/storage" - "google.golang.org/api/option" - - "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/common" - "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/uploader" -) - -const userAgent = "gcs-uploader" - -var ( - dir = flag.String("dir", ".", "Directory of files to upload") - location = flag.String("location", "", "Location of manifest file to upload; in the form gs://bucket/path/to/object") - workerCount = flag.Int("workers", 200, "The number of files to upload in parallel.") - help = flag.Bool("help", false, "If true, prints help text and exits.") -) - -func main() { - flag.Parse() - - if *help { - fmt.Println("Incrementally uploads source files to Google Cloud Storage") - flag.PrintDefaults() - return - } - - if *location == "" { - log.Fatalln("Must specify --location") - } - bucket, object, generation, err := common.ParseBucketObject(*location) - if err != nil { - log.Fatalf("parsing location from %q: %v", *location, err) - } - if generation != 0 { - log.Fatalln("cannot specify manifest file generation") - } - - ctx := context.Background() - client, err := storage.NewClient(ctx, option.WithUserAgent(userAgent)) - if err != nil { - log.Fatalf("Failed to create new GCS client: %v", err) - } - - u := uploader.New(ctx, realGCS{client}, realOS{}, bucket, object, *workerCount) - - filepath.Walk(*dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - u.Do(ctx, path, info) - return nil - }) - - if err := u.Done(ctx); err != nil { - log.Fatalf("Failed to upload: %v", err) - } -} - -// realGCS is a wrapper over the GCS client functions. -type realGCS struct { - client *storage.Client -} - -func (gp realGCS) NewWriter(ctx context.Context, bucket, object string) io.WriteCloser { - return gp.client.Bucket(bucket).Object(object). - If(storage.Conditions{DoesNotExist: true}). // Skip upload if already exists. - NewWriter(ctx) -} - -// realOS merely wraps the os package implementations. -type realOS struct{} - -func (realOS) EvalSymlinks(path string) (string, error) { return filepath.EvalSymlinks(path) } -func (realOS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } diff --git a/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/uploader/uploader.go b/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/uploader/uploader.go deleted file mode 100644 index 776820d4044..00000000000 --- a/vendor/github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/uploader/uploader.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2018 Google, Inc. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package uploader - -import ( - "context" - "crypto/sha1" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "sync" - - "google.golang.org/api/googleapi" - - "github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher/pkg/common" -) - -// Uploader encapsulates methods for uploading files incrementally and -// producing a source manifest. -type Uploader struct { - gcs GCS - os OS - bucket, manifestObject string - - manifest sync.Map - totalBytes, bytesSkipped int64 -} - -// OS allows us to inject dependencies to facilitate testing. -type OS interface { - EvalSymlinks(path string) (string, error) - Stat(path string) (os.FileInfo, error) -} - -// GCS allows us to inject dependencies to facilitate testing. -type GCS interface { - NewWriter(ctx context.Context, bucket, object string) io.WriteCloser -} - -type job struct { - path string - info os.FileInfo -} - -// New returns a new Uploader. -func New(ctx context.Context, gcs GCS, os OS, bucket, manifestObject string, numWorkers int) *Uploader { - return &Uploader{ - gcs: gcs, - os: os, - bucket: bucket, - manifestObject: manifestObject, - } -} - -// Wait blocks until ongoing uploads are complete, or until an error is -// encountered. -func (u *Uploader) Done(ctx context.Context) error { - uploaded := u.totalBytes - u.bytesSkipped - var incr float64 - if u.totalBytes != 0 { - incr = float64(100 * u.bytesSkipped / u.totalBytes) - } - fmt.Printf(` -****************************************************** -* Uploaded %d bytes (%.2f%% incremental) -****************************************************** -`, uploaded, incr) - return u.writeManifest(ctx) -} - -func (u *Uploader) Do(ctx context.Context, path string, info os.FileInfo) error { - // Follow symlinks. - if spath, err := u.os.EvalSymlinks(path); err != nil { - return err - } else if spath != path { - info, err = u.os.Stat(spath) - if err != nil { - return err - } - path = spath - } - - // Don't process dirs. - if info.IsDir() { - return nil - } - - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - - // Compute digest of file, and count bytes. - cw := &countWriter{} - h := sha1.New() - if _, err := io.Copy(io.MultiWriter(cw, h), f); err != nil { - return err - } - digest := fmt.Sprintf("%x", h.Sum(nil)) - - // Seek back to the beginning of the file, to write it to GCS. - // NB: The GCS client is responsible for skipping writes if the file - // already exists. - if _, err := f.Seek(0, 0); err != nil { - return err - } - wc := u.gcs.NewWriter(ctx, u.bucket, digest) - if _, err := io.Copy(wc, f); err != nil { - return err - } - - u.manifest.Store(path, common.ManifestItem{ - SourceURL: fmt.Sprintf("gs://%s/%s", u.bucket, digest), - Sha1Sum: digest, - FileMode: info.Mode(), - }) - - if err := wc.Close(); isAlreadyExists(err) { - u.bytesSkipped += cw.b - } else if err != nil { - return err - } - u.totalBytes += cw.b - return nil -} - -type countWriter struct { - b int64 -} - -func (c *countWriter) Write(b []byte) (int, error) { - c.b += int64(len(b)) - return len(b), nil -} - -func isAlreadyExists(err error) bool { - if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusPreconditionFailed { - return true - } - return false -} - -func (u *Uploader) writeManifest(ctx context.Context) error { - m := map[string]common.ManifestItem{} - u.manifest.Range(func(k, v interface{}) bool { - m[k.(string)] = v.(common.ManifestItem) - return true - }) - - wc := u.gcs.NewWriter(ctx, u.bucket, u.manifestObject) - if err := json.NewEncoder(wc).Encode(m); err != nil { - return err - } - if err := wc.Close(); err != nil { - return err - } - fmt.Printf("Wrote manifest object gs://%s/%s", u.bucket, u.manifestObject) - return nil -}