From d68cbf0529d3705b943894b72351b0d4bfa8f2b5 Mon Sep 17 00:00:00 2001 From: Hongxiang Jiang Date: Thu, 12 Sep 2024 15:29:44 +0000 Subject: [PATCH] internal/task: generate release artifacts in vscode-go prerelease Use RunCustomSteps method to trigger a few steps to build vscode-go release artifacts: - download go binary. - clone vscode-go repo and checkout specific commit. - execute npm ci and go run to build artifacts. - copy generated package extension and logs to gcs. The steps after this will use the returned CloudBuild struct (containing cloud build ID, project, gcs url) to fetch the artifacts and upload as github release assets. A local relui screenshot is at https://github.com/golang/vscode-go/issues/3500#issuecomment-2347320416 For golang/vscode-go#3500 Change-Id: Ia2987e1367cd6d76e019b4e576b6cc3e878ac751 Reviewed-on: https://go-review.googlesource.com/c/build/+/611945 LUCI-TryBot-Result: Go LUCI Reviewed-by: Dmitri Shuralyov Reviewed-by: Hyang-Ah Hana Kim --- internal/task/cloudbuild.go | 21 +++--- internal/task/fakes.go | 17 ++++- internal/task/releasevscodego.go | 67 +++++++++++++++++ internal/task/releasevscodego_test.go | 102 ++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 13 deletions(-) diff --git a/internal/task/cloudbuild.go b/internal/task/cloudbuild.go index d2fd6bd68..55a0f319d 100644 --- a/internal/task/cloudbuild.go +++ b/internal/task/cloudbuild.go @@ -78,25 +78,26 @@ func (c *RealCloudBuildClient) RunBuildTrigger(ctx context.Context, project, tri return CloudBuild{Project: project, ID: meta.Build.Id}, nil } -func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) { - const downloadGoScript = `#!/usr/bin/env bash +const cloudBuildClientScriptPrefix = `#!/usr/bin/env bash +set -eux +set -o pipefail +export PATH=/workspace/released_go/bin:$PATH +` + +const cloudBuildClientDownloadGoScript = `#!/usr/bin/env bash set -eux archive=$(wget -qO - 'https://go.dev/dl/?mode=json' | grep -Eo 'go.*linux-amd64.tar.gz' | head -n 1) wget -qO - https://go.dev/dl/${archive} | tar -xz mv go /workspace/released_go -` - const scriptPrefix = `#!/usr/bin/env bash -set -eux -set -o pipefail -export PATH=/workspace/released_go/bin:$PATH ` +func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) { steps := func(resultURL string) []*cloudbuildpb.BuildStep { // Cloud build loses directory structure when it saves artifacts, which is // a problem since (e.g.) we have multiple files named go.mod in the // tagging tasks. It's not very complicated, so reimplement it ourselves. var saveOutputsScript strings.Builder - saveOutputsScript.WriteString(scriptPrefix) + saveOutputsScript.WriteString(cloudBuildClientScriptPrefix) for _, out := range outputs { saveOutputsScript.WriteString(fmt.Sprintf("gsutil cp %q %q\n", out, resultURL+"/"+strings.TrimPrefix(out, "./"))) } @@ -113,11 +114,11 @@ export PATH=/workspace/released_go/bin:$PATH steps = append(steps, &cloudbuildpb.BuildStep{ Name: "bash", - Script: downloadGoScript, + Script: cloudBuildClientDownloadGoScript, }, &cloudbuildpb.BuildStep{ Name: "gcr.io/cloud-builders/gsutil", - Script: scriptPrefix + script, + Script: cloudBuildClientScriptPrefix + script, Dir: dir, }, &cloudbuildpb.BuildStep{ diff --git a/internal/task/fakes.go b/internal/task/fakes.go index 1f3bc18e1..52b98bdd5 100644 --- a/internal/task/fakes.go +++ b/internal/task/fakes.go @@ -632,9 +632,9 @@ case "$1" in esac ` -const fakeChown = ` +const fakeBinary = ` #!/bin/bash -eux -echo "chown change owner successful" +echo "this binary will always exit without any error" exit 0 ` @@ -646,7 +646,10 @@ func NewFakeCloudBuild(t *testing.T, gerrit *FakeGerrit, project string, allowed if err := os.WriteFile(filepath.Join(toolDir, "gsutil"), []byte(fakeGsutil), 0777); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(toolDir, "chown"), []byte(fakeChown), 0777); err != nil { + if err := os.WriteFile(filepath.Join(toolDir, "chown"), []byte(fakeBinary), 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(toolDir, "npm"), []byte(fakeBinary), 0777); err != nil { t.Fatal(err) } return &FakeCloudBuild{ @@ -756,6 +759,14 @@ func (cb *FakeCloudBuild) RunCustomSteps(ctx context.Context, steps func(resultU var gerritProject, fullScript string resultURL := "file://" + cb.t.TempDir() for _, step := range steps(resultURL) { + // Cloud Build support docker hub images like "bash". See more details: + // https://cloud.google.com/build/docs/interacting-with-dockerhub-images + // Currently, the Bash script is solely for downloading the Go binary. + // The RunScripts mock implementation provides the Go binary, allowing us + // to bypass the Bash script for now. + if step.Name == "bash" && step.Script == cloudBuildClientDownloadGoScript { + continue + } tool, found := strings.CutPrefix(step.Name, "gcr.io/cloud-builders/") if !found { return CloudBuild{}, fmt.Errorf("does not support custom image: %s", step.Name) diff --git a/internal/task/releasevscodego.go b/internal/task/releasevscodego.go index 4b1f656dd..aa2ded514 100644 --- a/internal/task/releasevscodego.go +++ b/internal/task/releasevscodego.go @@ -141,6 +141,8 @@ func (r *ReleaseVSCodeGoTasks) NewPrereleaseDefinition() *wf.Definition { _ = wf.Task1(wd, "create release milestone and issue", r.createReleaseMilestoneAndIssue, release, wf.After(verified)) branched := wf.Action2(wd, "create release branch", r.createReleaseBranch, release, prerelease, wf.After(verified)) + _ = wf.Task3(wd, "generate package extension (.vsix) for release candidate", r.generatePackageExtension, release, prerelease, revision, wf.After(verified)) + _ = wf.Action3(wd, "tag release candidate", r.tag, revision, release, prerelease, wf.After(branched)) return wd @@ -291,6 +293,71 @@ func (r *ReleaseVSCodeGoTasks) createReleaseBranch(ctx *wf.TaskContext, release return nil } +func (r *ReleaseVSCodeGoTasks) generatePackageExtension(ctx *wf.TaskContext, release releaseVersion, prerelease, revision string) (CloudBuild, error) { + steps := func(resultURL string) []*cloudbuildpb.BuildStep { + const packageScriptFmt = cloudBuildClientScriptPrefix + ` +export TAG_NAME=%s + +npm ci &> npm-output.log +go run tools/release/release.go package &> go-output.log +cat npm-output.log +cat go-output.log +` + + versionString := release.String() + if prerelease != "" { + versionString += "-" + prerelease + } + // The version inside of vsix does not have prefix "v". + vsix := fmt.Sprintf("go-%s.vsix", versionString[1:]) + + saveScript := cloudBuildClientScriptPrefix + for _, file := range []string{"npm-output.log", "go-output.log", vsix} { + saveScript += fmt.Sprintf("gsutil cp %s %s/%s\n", file, resultURL, file) + } + return []*cloudbuildpb.BuildStep{ + { + Name: "bash", + Script: cloudBuildClientDownloadGoScript, + }, + { + Name: "gcr.io/cloud-builders/git", + Args: []string{"clone", "https://go.googlesource.com/vscode-go", "vscode-go"}, + }, + { + Name: "gcr.io/cloud-builders/git", + Args: []string{"checkout", revision}, + Dir: "vscode-go", + }, + { + Name: "gcr.io/cloud-builders/npm", + Script: fmt.Sprintf(packageScriptFmt, versionString), + Dir: "vscode-go/extension", + }, + { + Name: "gcr.io/cloud-builders/gsutil", + Script: saveScript, + Dir: "vscode-go/extension", + }, + } + } + + build, err := r.CloudBuild.RunCustomSteps(ctx, steps) + if err != nil { + return CloudBuild{}, err + } + + outputs, err := buildToOutputs(ctx, r.CloudBuild, build) + if err != nil { + return CloudBuild{}, err + } + + ctx.Printf("the output from npm ci:\n%s\n", outputs["npm-output.log"]) + ctx.Printf("the output from package generation:\n%s\n", outputs["go-output.log"]) + + return build, nil +} + // determineReleaseVersion determines the release version for the upcoming // stable release of vscode-go by examining all existing tags in the repository. // diff --git a/internal/task/releasevscodego_test.go b/internal/task/releasevscodego_test.go index 83715b5f4..6c9447d60 100644 --- a/internal/task/releasevscodego_test.go +++ b/internal/task/releasevscodego_test.go @@ -7,6 +7,7 @@ package task import ( "context" "fmt" + "io" "testing" "github.com/google/go-github/v48/github" @@ -425,6 +426,107 @@ esac } } +func TestGeneratePackageExtension(t *testing.T) { + mustHaveShell(t) + testcases := []struct { + name string + release releaseVersion + prerelease string + rc int + wantErr bool + }{ + { + name: "test failed, return error", + release: releaseVersion{0, 1, 0}, + prerelease: "rc-1", + rc: 1, + wantErr: true, + }, + { + name: "test passed, return nil", + release: releaseVersion{0, 2, 3}, + prerelease: "", + rc: 0, + wantErr: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + vscodego := NewFakeRepo(t, "vscode-go") + commit := vscodego.Commit(map[string]string{ + "go.mod": "module github.com/golang/vscode-go\n", + "go.sum": "\n", + "extension/tools/release/release.go": "foo", + }) + + gerrit := NewFakeGerrit(t, vscodego) + ctx := &workflow.TaskContext{ + Context: context.Background(), + Logger: &testLogger{t, ""}, + } + + version := tc.release.String()[1:] + if tc.prerelease != "" { + version += tc.prerelease + } + // fakeGo write "bar" content to go-${version}.vsix file and "foo" content + // to README.md when executed go run tools/release/release.go. + var fakeGo = fmt.Sprintf(`#!/bin/bash -exu + +case "$1" in +"run") + echo "writing content to vsix and README.md" + echo -n "bar" > go-%s.vsix + echo -n "foo" > README.md + exit %v + ;; +*) + echo unexpected command $@ + exit 1 + ;; +esac +`, version, tc.rc) + + cloudbuild := NewFakeCloudBuild(t, gerrit, "vscode-go", nil, fakeGo) + tasks := &ReleaseVSCodeGoTasks{ + Gerrit: gerrit, + CloudBuild: cloudbuild, + } + + cb, err := tasks.generatePackageExtension(ctx, tc.release, tc.prerelease, commit) + if tc.wantErr && err == nil { + t.Errorf("generateArtifacts(%s, %s, %s) should return error but return nil", tc.release, tc.prerelease, commit) + } else if !tc.wantErr && err != nil { + t.Errorf("generateArtifacts(%s, %s, %s) should return nil but return err: %v", tc.release, tc.prerelease, commit, err) + } + + if !tc.wantErr { + path := fmt.Sprintf("go-%s.vsix", version) + resultFS, err := cloudbuild.ResultFS(ctx, cb) + if err != nil { + t.Fatal(err) + } + + f, err := resultFS.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + got, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + if string(got) != "bar" { + t.Errorf("generateArtifacts(%s, %s, %s) write content %q to %s, want %q", tc.release, tc.prerelease, commit, got, path, "bar") + } + } + }) + } +} + func TestDetermineInsiderVersion(t *testing.T) { testcases := []struct { name string