feat: use MS Go fork for windows and linux builds

Co-authored-by: JSON <jason.luong@snyk.io> Co-authored-by: Avishag Israeli <44115709+Avishagp@users.noreply.github.com>
snyk · Aug 9, 2023 · 3385306 · 3385306
1 parent 20ddac3
commit 3385306
Show file tree

Hide file tree

Showing 5 changed files with 217 additions and 53 deletions.
diff --git a/.circleci/chocolatey.config b/.circleci/chocolatey.config
@@ -3,6 +3,5 @@
     <package id="maven" version="3.8.2" />
     <package id="make" version="4.4.1" />
     <package id="gradle" version="8.1.1" />
-    <package id="go" version="1.20.6" />
     <package id="python3" version="3.11.4" />
 </packages>
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,7 +3,6 @@ version: '2.1'
 orbs:
   prodsec: snyk/prodsec-orb@1.0
   snyk: snyk/snyk@1.7.0
-  go: circleci/go@1.7.1
   win: circleci/windows@5.0
   aws-cli: circleci/aws-cli@2.0.3
   gh: circleci/github-cli@2.1.0
@@ -22,11 +21,6 @@ executors:
       - image: bastiandoetsch209/cli-build-arm64:20230806-080507
     working_directory: /mnt/ramdisk/snyk
     resource_class: arm.large
-  linux-amd64:
-    machine:
-      image: ubuntu-2204:2023.02.1
-    working_directory: /mnt/ramdisk/snyk
-    resource_class: large
   linux-ubuntu-mantic-amd64:
     docker:
       - image: ubuntu:mantic
@@ -77,8 +71,63 @@ executors:
       # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
       xcode: '14.3.1'
     resource_class: macos.m1.large.gen1
+  win-amd64:
+    machine:
+      image: windows-server-2022-gui:2023.07.1
+    resource_class: windows.large
+    shell: powershell
 
 commands:
+  install-go:
+    parameters:
+      go_os:
+        type: string
+      go_target_os:
+        type: string
+      go_arch:
+        type: string
+      base_url:
+        type: string
+      extraction_path:
+        type: string
+      cache_key_file:
+        type: string
+        default: go_cache_key.txt
+    steps:
+      - run:
+          name: Create Cache Key
+          command: |
+            echo << parameters.extraction_path >>-<< parameters.base_url >>-<< parameters.go_target_os >>-<< parameters.go_arch >>-<< pipeline.parameters.go_version >> > << parameters.cache_key_file >>
+            cat << parameters.cache_key_file >>
+      - restore_cache:
+          name: Restoring go binary cache
+          keys:
+            - go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }}
+      - run:
+          name: Download go binary
+          command: python ./scripts/download_go.py << pipeline.parameters.go_version >> --go_os=<< parameters.go_os >> --go_arch=<< parameters.go_arch >> --base_url=<< parameters.base_url >> --extraction_path=<< parameters.extraction_path >>
+      - save_cache:
+          name: Caching go binary
+          key: go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }}
+          paths:
+            - << parameters.extraction_path >>/go
+      - unless:
+          condition:
+            equal: ['windows', << parameters.go_os >>]
+          steps:
+            - run:
+                name: Install go binary
+                command: echo "export PATH=$(realpath << parameters.extraction_path >>/go/bin):\$PATH" >> "$BASH_ENV"
+      - when:
+          condition:
+            equal: ['windows', << parameters.go_os >>]
+          steps:
+            - run:
+                name: Install go binary
+                command: |
+                  New-Item -Path $profile -ItemType File -Force
+                  '$Env:Path = "<< parameters.extraction_path >>\go\bin;" + $Env:Path' >> $profile
+
   install-deps-windows-full:
     steps:
       - restore_cache:
@@ -159,6 +208,9 @@ commands:
           at: .
 
 parameters:
+  go_base_url:
+    type: string
+    default: https://aka.ms/golang/release/latest/
   go_version:
     type: string
     # https://go.dev/doc/devel/release
@@ -214,46 +266,53 @@ workflows:
 
       - build-artifact:
           name: build linux amd64
+          go_target_os: linux
           go_os: linux
           go_arch: amd64
+          go_base_url: << pipeline.parameters.go_base_url >>
           executor: docker-amd64
-          artifact: snyk-linux
           requires:
             - prepare-build
 
       - build-artifact:
           name: build linux arm64
+          go_target_os: linux
           go_os: linux
           go_arch: arm64
+          go_base_url: << pipeline.parameters.go_base_url >>
           executor: docker-arm64
-          artifact: snyk-linux-arm64
           requires:
             - prepare-build
 
       - build-artifact:
           name: build alpine amd64
-          go_os: alpine
+          go_target_os: alpine
+          go_os: linux
           go_arch: amd64
           executor: docker-amd64
-          artifact: snyk-alpine
           c_compiler: /usr/bin/musl-gcc
           requires:
             - prepare-build
 
       - build-artifact:
           name: build macOS amd64
+          go_target_os: darwin
           go_os: darwin
           go_arch: amd64
           executor: macos-amd64
-          artifact: snyk-macos
           requires:
             - prepare-build
 
-      - build-windows-artifact:
-          context: snyk-windows-signing
+      - build-artifact:
           name: build windows amd64
+          go_target_os: windows
           go_os: windows
           go_arch: amd64
+          go_base_url: << pipeline.parameters.go_base_url >>
+          install_deps_extension: windows-full
+          install_path: 'C:\'
+          executor: win-amd64
+          context: snyk-windows-signing
           requires:
             - prepare-build
 
@@ -563,60 +622,46 @@ jobs:
             - binary-releases/snyk-fix.tgz
             - binary-releases/snyk-protect.tgz
 
-  build-windows-artifact:
-    parameters:
-      go_os:
-        type: string
-      go_arch:
-        type: string
-    executor:
-      name: win/default
-      size: large
-    steps:
-      - prepare-workspace
-      - install-deps-windows-full
-      - run:
-          name: Build Windows
-          shell: powershell
-          command: make build GOOS=windows GOARCH=amd64
-          environment:
-            CGO_ENABLED: 1
-      - store_artifacts:
-          path: binary-releases
-      - persist_to_workspace:
-          root: .
-          paths:
-            - binary-releases/snyk-*
-            - binary-releases/version
-
   build-artifact:
     parameters:
       go_os:
         type: string
+      go_target_os:
+        type: string
       go_arch:
         type: string
       c_compiler:
         type: string
         default: ''
       executor:
         type: string
-      artifact:
+      go_base_url:
+        type: string
+        default: 'https://go.dev/dl/'
+      install_deps_extension:
         type: string
+        default: 'noop'
+      install_path:
+        type: string
+        default: '.'
     executor: << parameters.executor >>
-    environment:
-      HOSTTYPE: << parameters.go_arch >>
     steps:
       - prepare-workspace
-      - go/install:
-          version: << pipeline.parameters.go_version >>
+      - install-deps-<< parameters.install_deps_extension >>
+      - install-go:
+          go_os: << parameters.go_os >>
+          go_target_os: << parameters.go_target_os >>
+          go_arch: << parameters.go_arch >>
+          base_url: << parameters.go_base_url >>
+          extraction_path: << parameters.install_path >>
       - restore_cache:
           key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }}
       - run:
-          name: Build << parameters.go_os >>/<< parameters.go_arch >>
+          name: Build << parameters.go_target_os >>/<< parameters.go_arch >>
           environment:
             CC: << parameters.c_compiler >>
             CGO_ENABLED: 1
-          command: make build GOOS=<< parameters.go_os >> GOARCH=<< parameters.go_arch >>
+          command: make build GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >>
       - save_cache:
           key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }}
           paths: [/home/circleci/go/pkg/mod]

diff --git a/Makefile b/Makefile
@@ -13,10 +13,10 @@ BINARY_RELEASES_FOLDER_TS_CLI = binary-releases
 BINARY_OUTPUT_FOLDER = binary-releases
 SHASUM_CMD = shasum
 GOHOSTOS = $(shell go env GOHOSTOS)
-PYTHON = python
+export PYTHON = python
 
 PYTHON_VERSION = $(shell python3 --version)
-ifdef (PYTHON_VERSION)
+ifneq (, $(PYTHON_VERSION))
 	PYTHON = python3
 endif
 
@@ -60,9 +60,12 @@ prepack: $(BINARY_OUTPUT_FOLDER)/version
 	cd $(BINARY_WRAPPER_DIR); npm version "$(shell cat $(WORKING_DIR)/$(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --include-workspace-root
 	npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts
 
-.PHONY: clean-prepack
-clean-prepack:
+.PHONY: clean-package-files
+clean-package-files:
 	git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json $(BINARY_WRAPPER_DIR)/package.json $(BINARY_WRAPPER_DIR)/package-lock.json
+
+.PHONY: clean-prepack
+clean-prepack: clean-package-files
 	rm -f prepack
 
 .PHONY: clean-ts
@@ -202,6 +205,7 @@ pre-build: pre-build-binary-wrapper $(BINARY_RELEASES_FOLDER_TS_CLI)
 .PHONY: build
 build: pre-build
 	@cd $(EXTENSIBLE_CLI_DIR); $(MAKE) build-full install bindir=$(WORKING_DIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1
+	@$(MAKE) clean-package-files
 
 .PHONY: sign
 sign:

diff --git a/cliv2/Makefile b/cliv2/Makefile
@@ -5,6 +5,8 @@ GOOS = $(shell go env GOOS)
 GOARCH = $(shell go env GOARCH)
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
+GOEXPERIMENT_DEFAULT_CRYPTO = opensslcrypto
+GOEXPERIMENT =
 HASH = sha
 HASH_ALGORITHM = 256
 CLI_V2_VERSION_TAG = 
@@ -99,6 +101,12 @@ ifeq ($(GOHOSTOS), windows)
 	SHASUM_CMD = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/shasum.ps1
 	SIGN_SCRIPT = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/sign_$(GOHOSTOS).ps1
 	ISSIGNED_SCRIPT = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/issigned_$(GOHOSTOS).ps1
+	GOEXPERIMENT_DEFAULT_CRYPTO = cngcrypto
+endif
+
+GO_SYSTEMCRYPTO_SUPPORTED = $(shell GOEXPERIMENT=$(GOEXPERIMENT_DEFAULT_CRYPTO) go version 2> /dev/null)
+ifneq (,$(GO_SYSTEMCRYPTO_SUPPORTED))
+	GOEXPERIMENT = $(GOEXPERIMENT_DEFAULT_CRYPTO)
 endif
 
 # some make file variables
@@ -159,13 +167,20 @@ _validate_sha_v1: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING)
 .PHONY: dependencies
 dependencies: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) _validate_sha_v1
 
+.PHONY: summary
+summary:
+	@echo "$(LOG_PREFIX) Pre-paring to build"
+	@echo "$(LOG_PREFIX)  Using Go:        $(shell which go)"
+	@echo "$(LOG_PREFIX)  Building Binary: $(BUILD_DIR)/$(V2_EXECUTABLE_NAME)"
+	@echo "$(LOG_PREFIX)  GOEXPERIMENT:    $(GOEXPERIMENT)"
+
 # prepare the workspace and cache global parameters
 .PHONY: configure
-configure: $(V2_DIRECTORY)/cliv2.version $(CACHE_DIR) $(CACHE_DIR)/version.mk $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses
+configure: summary $(V2_DIRECTORY)/cliv2.version $(CACHE_DIR) $(CACHE_DIR)/version.mk $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses
 
 $(BUILD_DIR)/$(V2_EXECUTABLE_NAME): $(BUILD_DIR) $(SRCS)
 	@echo "$(LOG_PREFIX) Building ( $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) )"
-	@GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
+	@GOEXPERIMENT=$(GOEXPERIMENT) GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
 
 .PHONY: build
 build: configure $(BUILD_DIR)/$(V2_EXECUTABLE_NAME)