diff --git a/.circleci/chocolatey.config b/.circleci/chocolatey.config
index afa94a7dad..e3dc238e11 100644
--- a/.circleci/chocolatey.config
+++ b/.circleci/chocolatey.config
@@ -3,6 +3,5 @@
-
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9ea2a6a974..cbaae697e6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,7 +3,6 @@ version: '2.1'
orbs:
prodsec: snyk/prodsec-orb@1.0
snyk: snyk/snyk@1.7.0
- go: circleci/go@1.7.1
win: circleci/windows@5.0
aws-cli: circleci/aws-cli@2.0.3
gh: circleci/github-cli@2.1.0
@@ -22,11 +21,6 @@ executors:
- image: bastiandoetsch209/cli-build-arm64:20230806-080507
working_directory: /mnt/ramdisk/snyk
resource_class: arm.large
- linux-amd64:
- machine:
- image: ubuntu-2204:2023.02.1
- working_directory: /mnt/ramdisk/snyk
- resource_class: large
linux-ubuntu-mantic-amd64:
docker:
- image: ubuntu:mantic
@@ -77,8 +71,63 @@ executors:
# https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
xcode: '14.3.1'
resource_class: macos.m1.large.gen1
+ win-amd64:
+ machine:
+ image: windows-server-2022-gui:2023.07.1
+ resource_class: windows.large
+ shell: powershell
commands:
+ install-go:
+ parameters:
+ go_os:
+ type: string
+ go_target_os:
+ type: string
+ go_arch:
+ type: string
+ base_url:
+ type: string
+ extraction_path:
+ type: string
+ cache_key_file:
+ type: string
+ default: go_cache_key.txt
+ steps:
+ - run:
+ name: Create Cache Key
+ command: |
+ echo << parameters.extraction_path >>-<< parameters.base_url >>-<< parameters.go_target_os >>-<< parameters.go_arch >>-<< pipeline.parameters.go_version >> > << parameters.cache_key_file >>
+ cat << parameters.cache_key_file >>
+ - restore_cache:
+ name: Restoring go binary cache
+ keys:
+ - go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }}
+ - run:
+ name: Download go binary
+ command: python ./scripts/download_go.py << pipeline.parameters.go_version >> --go_os=<< parameters.go_os >> --go_arch=<< parameters.go_arch >> --base_url=<< parameters.base_url >> --extraction_path=<< parameters.extraction_path >>
+ - save_cache:
+ name: Caching go binary
+ key: go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }}
+ paths:
+ - << parameters.extraction_path >>/go
+ - unless:
+ condition:
+ equal: ['windows', << parameters.go_os >>]
+ steps:
+ - run:
+ name: Install go binary
+ command: echo "export PATH=$(realpath << parameters.extraction_path >>/go/bin):\$PATH" >> "$BASH_ENV"
+ - when:
+ condition:
+ equal: ['windows', << parameters.go_os >>]
+ steps:
+ - run:
+ name: Install go binary
+ command: |
+ New-Item -Path $profile -ItemType File -Force
+ '$Env:Path = "<< parameters.extraction_path >>\go\bin;" + $Env:Path' >> $profile
+
install-deps-windows-full:
steps:
- restore_cache:
@@ -159,6 +208,9 @@ commands:
at: .
parameters:
+ go_base_url:
+ type: string
+ default: https://aka.ms/golang/release/latest/
go_version:
type: string
# https://go.dev/doc/devel/release
@@ -214,46 +266,53 @@ workflows:
- build-artifact:
name: build linux amd64
+ go_target_os: linux
go_os: linux
go_arch: amd64
+ go_base_url: << pipeline.parameters.go_base_url >>
executor: docker-amd64
- artifact: snyk-linux
requires:
- prepare-build
- build-artifact:
name: build linux arm64
+ go_target_os: linux
go_os: linux
go_arch: arm64
+ go_base_url: << pipeline.parameters.go_base_url >>
executor: docker-arm64
- artifact: snyk-linux-arm64
requires:
- prepare-build
- build-artifact:
name: build alpine amd64
- go_os: alpine
+ go_target_os: alpine
+ go_os: linux
go_arch: amd64
executor: docker-amd64
- artifact: snyk-alpine
c_compiler: /usr/bin/musl-gcc
requires:
- prepare-build
- build-artifact:
name: build macOS amd64
+ go_target_os: darwin
go_os: darwin
go_arch: amd64
executor: macos-amd64
- artifact: snyk-macos
requires:
- prepare-build
- - build-windows-artifact:
- context: snyk-windows-signing
+ - build-artifact:
name: build windows amd64
+ go_target_os: windows
go_os: windows
go_arch: amd64
+ go_base_url: << pipeline.parameters.go_base_url >>
+ install_deps_extension: windows-full
+ install_path: 'C:\'
+ executor: win-amd64
+ context: snyk-windows-signing
requires:
- prepare-build
@@ -563,36 +622,12 @@ jobs:
- binary-releases/snyk-fix.tgz
- binary-releases/snyk-protect.tgz
- build-windows-artifact:
- parameters:
- go_os:
- type: string
- go_arch:
- type: string
- executor:
- name: win/default
- size: large
- steps:
- - prepare-workspace
- - install-deps-windows-full
- - run:
- name: Build Windows
- shell: powershell
- command: make build GOOS=windows GOARCH=amd64
- environment:
- CGO_ENABLED: 1
- - store_artifacts:
- path: binary-releases
- - persist_to_workspace:
- root: .
- paths:
- - binary-releases/snyk-*
- - binary-releases/version
-
build-artifact:
parameters:
go_os:
type: string
+ go_target_os:
+ type: string
go_arch:
type: string
c_compiler:
@@ -600,23 +635,33 @@ jobs:
default: ''
executor:
type: string
- artifact:
+ go_base_url:
+ type: string
+ default: 'https://go.dev/dl/'
+ install_deps_extension:
type: string
+ default: 'noop'
+ install_path:
+ type: string
+ default: '.'
executor: << parameters.executor >>
- environment:
- HOSTTYPE: << parameters.go_arch >>
steps:
- prepare-workspace
- - go/install:
- version: << pipeline.parameters.go_version >>
+ - install-deps-<< parameters.install_deps_extension >>
+ - install-go:
+ go_os: << parameters.go_os >>
+ go_target_os: << parameters.go_target_os >>
+ go_arch: << parameters.go_arch >>
+ base_url: << parameters.go_base_url >>
+ extraction_path: << parameters.install_path >>
- restore_cache:
key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }}
- run:
- name: Build << parameters.go_os >>/<< parameters.go_arch >>
+ name: Build << parameters.go_target_os >>/<< parameters.go_arch >>
environment:
CC: << parameters.c_compiler >>
CGO_ENABLED: 1
- command: make build GOOS=<< parameters.go_os >> GOARCH=<< parameters.go_arch >>
+ command: make build GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >>
- save_cache:
key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }}
paths: [/home/circleci/go/pkg/mod]
diff --git a/Makefile b/Makefile
index 30faff3467..676d66daad 100644
--- a/Makefile
+++ b/Makefile
@@ -13,10 +13,10 @@ BINARY_RELEASES_FOLDER_TS_CLI = binary-releases
BINARY_OUTPUT_FOLDER = binary-releases
SHASUM_CMD = shasum
GOHOSTOS = $(shell go env GOHOSTOS)
-PYTHON = python
+export PYTHON = python
PYTHON_VERSION = $(shell python3 --version)
-ifdef (PYTHON_VERSION)
+ifneq (, $(PYTHON_VERSION))
PYTHON = python3
endif
@@ -60,9 +60,12 @@ prepack: $(BINARY_OUTPUT_FOLDER)/version
cd $(BINARY_WRAPPER_DIR); npm version "$(shell cat $(WORKING_DIR)/$(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --include-workspace-root
npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts
-.PHONY: clean-prepack
-clean-prepack:
+.PHONY: clean-package-files
+clean-package-files:
git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json $(BINARY_WRAPPER_DIR)/package.json $(BINARY_WRAPPER_DIR)/package-lock.json
+
+.PHONY: clean-prepack
+clean-prepack: clean-package-files
rm -f prepack
.PHONY: clean-ts
@@ -202,6 +205,7 @@ pre-build: pre-build-binary-wrapper $(BINARY_RELEASES_FOLDER_TS_CLI)
.PHONY: build
build: pre-build
@cd $(EXTENSIBLE_CLI_DIR); $(MAKE) build-full install bindir=$(WORKING_DIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1
+ @$(MAKE) clean-package-files
.PHONY: sign
sign:
diff --git a/cliv2/Makefile b/cliv2/Makefile
index 97a009fb56..1ac4c1b982 100644
--- a/cliv2/Makefile
+++ b/cliv2/Makefile
@@ -5,6 +5,8 @@ GOOS = $(shell go env GOOS)
GOARCH = $(shell go env GOARCH)
GOHOSTOS = $(shell go env GOHOSTOS)
GOHOSTARCH = $(shell go env GOHOSTARCH)
+GOEXPERIMENT_DEFAULT_CRYPTO = opensslcrypto
+GOEXPERIMENT =
HASH = sha
HASH_ALGORITHM = 256
CLI_V2_VERSION_TAG =
@@ -99,6 +101,12 @@ ifeq ($(GOHOSTOS), windows)
SHASUM_CMD = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/shasum.ps1
SIGN_SCRIPT = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/sign_$(GOHOSTOS).ps1
ISSIGNED_SCRIPT = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/issigned_$(GOHOSTOS).ps1
+ GOEXPERIMENT_DEFAULT_CRYPTO = cngcrypto
+endif
+
+GO_SYSTEMCRYPTO_SUPPORTED = $(shell GOEXPERIMENT=$(GOEXPERIMENT_DEFAULT_CRYPTO) go version 2> /dev/null)
+ifneq (,$(GO_SYSTEMCRYPTO_SUPPORTED))
+ GOEXPERIMENT = $(GOEXPERIMENT_DEFAULT_CRYPTO)
endif
# some make file variables
@@ -159,13 +167,20 @@ _validate_sha_v1: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING)
.PHONY: dependencies
dependencies: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) _validate_sha_v1
+.PHONY: summary
+summary:
+ @echo "$(LOG_PREFIX) Pre-paring to build"
+ @echo "$(LOG_PREFIX) Using Go: $(shell which go)"
+ @echo "$(LOG_PREFIX) Building Binary: $(BUILD_DIR)/$(V2_EXECUTABLE_NAME)"
+ @echo "$(LOG_PREFIX) GOEXPERIMENT: $(GOEXPERIMENT)"
+
# prepare the workspace and cache global parameters
.PHONY: configure
-configure: $(V2_DIRECTORY)/cliv2.version $(CACHE_DIR) $(CACHE_DIR)/version.mk $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses
+configure: summary $(V2_DIRECTORY)/cliv2.version $(CACHE_DIR) $(CACHE_DIR)/version.mk $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses
$(BUILD_DIR)/$(V2_EXECUTABLE_NAME): $(BUILD_DIR) $(SRCS)
@echo "$(LOG_PREFIX) Building ( $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) )"
- @GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
+ @GOEXPERIMENT=$(GOEXPERIMENT) GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
.PHONY: build
build: configure $(BUILD_DIR)/$(V2_EXECUTABLE_NAME)
diff --git a/scripts/download_go.py b/scripts/download_go.py
new file mode 100644
index 0000000000..e954d1afa3
--- /dev/null
+++ b/scripts/download_go.py
@@ -0,0 +1,101 @@
+import argparse
+import hashlib
+import os
+import platform
+import urllib.request
+import tarfile
+import zipfile
+import tempfile
+
+# determine go binary to download and extract
+def get_go_binary_name(go_os, go_arch, go_version):
+ filename = "go" + go_version + "." + go_os + "-" + go_arch
+
+ # determine file extension
+ if go_os == "windows":
+ filename += ".zip"
+ else:
+ filename += ".tar.gz"
+
+ # e.g. go1.20.linux-amd64.tar.gz
+ return filename
+
+# download go binary
+def download_go_binary(filepath, filename, base_url):
+ url = base_url + filename
+ tmp_file = os.path.join(filepath, filename)
+
+ print("Downloading " + url + " to " + tmp_file)
+
+ try:
+ urllib.request.urlretrieve(url, tmp_file)
+ print("Download complete")
+ except urllib.error.URLError as e:
+ print("Error while downloading the file:", e)
+
+ return tmp_file
+
+# extract tar file
+def extract_tar(tar_gz_file, extract_path):
+ print("Extracting " + tar_gz_file + " to " + extract_path)
+ try:
+ with tarfile.open(tar_gz_file, "r:gz") as tar:
+ tar.extractall(path=extract_path)
+ print("Extraction complete")
+ except tarfile.TarError as e:
+ print("Error while extracting the tar file:", e)
+
+# unzip zip file
+def unzip_file(zip_file, extract_path):
+ print("Unzipping " + zip_file + " to " + extract_path)
+ try:
+ with zipfile.ZipFile(zip_file, 'r') as zip_ref:
+ zip_ref.extractall(extract_path)
+ print("Unzipping complete")
+ except zipfile.BadZipFile as e:
+ print("Error while unzipping the file:", e)
+
+# setup argparse
+def init_argparse():
+ parser = argparse.ArgumentParser(
+ prog="download_go",
+ description="Download and install a specific version of Go."
+ )
+ parser.add_argument("version", help="Version of Go to download (e.g., 1.20)")
+ parser.add_argument("--go_os",
+ help="OS to download for (e.g., linux, windows, darwin)",
+ choices=["linux", "windows", "darwin"],
+ required=True)
+ parser.add_argument("--go_arch",
+ help="Architecture to download for (e.g., amd64, arm64)",
+ choices=["amd64", "arm64", "armv6l"],
+ required=True)
+ parser.add_argument("--base_url",
+ help="Base URL to download from (e.g., https://go.dev/dl/)",
+ choices=["https://go.dev/dl/", "https://aka.ms/golang/release/latest/"],
+ default="https://go.dev/dl/")
+ parser.add_argument("--extraction_path",
+ help="Path to download the file to (e.g., /tmp)",
+ default=os.getcwd())
+
+ return parser.parse_args()
+
+if __name__ == "__main__":
+ args = init_argparse()
+
+ # check if go is cached, before trying to download and extract it
+ file_dir = os.path.join(os.path.abspath(args.extraction_path), "go")
+ if os.path.exists(file_dir):
+ print("Restored from cache, skipping download and extraction")
+ exit(0)
+
+ # create temporary download path
+ download_path = tempfile.gettempdir()
+
+ # download and extract the go binary
+ binary_name = get_go_binary_name(args.go_os, args.go_arch, args.version)
+ binary_file = download_go_binary(download_path, binary_name, args.base_url)
+ if binary_name.endswith(".zip"):
+ unzip_file(binary_file, os.path.abspath(args.extraction_path))
+ else:
+ extract_tar(binary_file, os.path.abspath(args.extraction_path))