From 34e733c46b108c8f93194858a557b17965746cbc Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 6 Dec 2023 02:33:12 +0400 Subject: [PATCH 01/37] remove some jobs --- .gitlab-ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8a91568ccfc..eb01e5d3b428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,10 +119,10 @@ default: # .forklift-cache: before_script: - - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"' + - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' - chmod +x forklift - mkdir .forklift - - cp $FL_FORKLIFT_CONFIG .forklift/config.toml + - cp $FL_FORKLIFT_CONFIG_ALT .forklift/config.toml - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*} - shopt -s expand_aliases - export PATH=$PATH:$(pwd) @@ -135,7 +135,7 @@ default: - rm -f forklift.sock - forklift clean # - - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}" + - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX" .common-refs: @@ -214,12 +214,6 @@ include: - .gitlab/pipeline/test.yml # build jobs - .gitlab/pipeline/build.yml - # short-benchmarks jobs - - .gitlab/pipeline/short-benchmarks.yml - # publish jobs - - .gitlab/pipeline/publish.yml - # zombienet jobs - - .gitlab/pipeline/zombienet.yml # ci image - project: parity/infrastructure/ci_cd/shared ref: main From 2a4c102e0f7ce39a5b0a4bb34cb0d8fe929b96b1 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 7 Dec 2023 19:51:33 +0400 Subject: [PATCH 02/37] alt bybass var name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb01e5d3b428..3fc5029b238d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,7 +127,7 @@ default: - shopt -s expand_aliases - export PATH=$PATH:$(pwd) - | - if [ "$FORKLIFT_BYPASS" != "true" ]; then + if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'" alias cargo="forklift cargo" fi From e63edf346c5c6cda30ba20110c0d27e20481c2bb Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 7 Dec 2023 20:19:22 +0400 Subject: [PATCH 03/37] trigger --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 924f4b400ee6..19f2e9fe681a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,7 +131,6 @@ default: echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'" alias cargo="forklift cargo" fi - - ls -al - rm -f forklift.sock - forklift clean # From 945acee332d279228eef3d40fb44ae8fd3ab75eb Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 4 Jan 2024 17:14:11 +0400 Subject: [PATCH 04/37] add forklift config to home dir --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19f2e9fe681a..3cf93582c996 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,6 +123,8 @@ default: - chmod +x forklift - mkdir .forklift - cp $FL_FORKLIFT_CONFIG_ALT .forklift/config.toml + - mkdir ~/.forklift + - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*} - shopt -s expand_aliases - export PATH=$PATH:$(pwd) From a75532155dcd1c01eacbd0a429ebec577c84171b Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Sat, 27 Jan 2024 14:20:33 +0300 Subject: [PATCH 05/37] remove package suffix --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f89975e73bb..367fac540ff4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,8 +124,6 @@ default: - mkdir .forklift - cp $FL_FORKLIFT_CONFIG_ALT .forklift/config.toml - mkdir ~/.forklift - - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*} - shopt -s expand_aliases - export PATH=$PATH:$(pwd) - | @@ -134,7 +132,6 @@ default: alias cargo="forklift cargo" fi - rm -f forklift.sock - - forklift clean # - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX" From 18d41824d98b585b317310126b1cfc0e87161765 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 6 Feb 2024 01:31:55 +0300 Subject: [PATCH 06/37] home config --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 367fac540ff4..d8236e87624c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,8 +122,8 @@ default: - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' - chmod +x forklift - mkdir .forklift - - cp $FL_FORKLIFT_CONFIG_ALT .forklift/config.toml - mkdir ~/.forklift + - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - shopt -s expand_aliases - export PATH=$PATH:$(pwd) - | From 83d4cf622872339d6e303d0cea5c179851201497 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 6 Feb 2024 05:18:53 +0300 Subject: [PATCH 07/37] forklift for check each crate --- .gitlab/check-each-crate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/check-each-crate.py b/.gitlab/check-each-crate.py index da2eaad36c52..9b654f8071ac 100755 --- a/.gitlab/check-each-crate.py +++ b/.gitlab/check-each-crate.py @@ -55,7 +55,7 @@ print(f"Checking {crates[crate][0]}", file=sys.stderr) - res = subprocess.run(["cargo", "check", "--locked"], cwd = crates[crate][1]) + res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1]) if res.returncode != 0: sys.exit(1) From 6fa18ab6810aa3e947cb0d01707bbf0aa3666e4c Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 17:48:38 +0300 Subject: [PATCH 08/37] forklift cargo function --- .gitlab-ci.yml | 7 ++----- .gitlab/pipeline/build.yml | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8236e87624c..34d2c2fa0127 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,17 +121,14 @@ default: before_script: - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' - chmod +x forklift - - mkdir .forklift - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - - shopt -s expand_aliases - export PATH=$PATH:$(pwd) - | if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'" - alias cargo="forklift cargo" + echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo $1; }" + function cargo() { forklift cargo $1; } fi - - rm -f forklift.sock # - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX" diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 002206e328cf..740a10d4472b 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -314,6 +314,7 @@ build-linux-substrate: # tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589 - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" + - !reference [.forklift-cache, before_script] script: - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p staging-node-cli - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate From 9ab9d094afbcb4cad6817aea0e4e880d00b9ae14 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 17:55:18 +0300 Subject: [PATCH 09/37] @ --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34d2c2fa0127..33d5ee813caf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,7 +126,7 @@ default: - export PATH=$PATH:$(pwd) - | if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo $1; }" + echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo $@; }" function cargo() { forklift cargo $1; } fi # From 935fb2dc3565e030440c155d5801ffb0b1ab4258 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 18:07:51 +0300 Subject: [PATCH 10/37] "" --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d5ee813caf..c994b05c40b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,7 +126,7 @@ default: - export PATH=$PATH:$(pwd) - | if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo $@; }" + echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo "$@" }" function cargo() { forklift cargo $1; } fi # From 49025953d130a855a55875c5518edfcdaa5cd5cc Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 18:16:52 +0300 Subject: [PATCH 11/37] facepalm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c994b05c40b7..871c821a72d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,7 +127,7 @@ default: - | if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo "$@" }" - function cargo() { forklift cargo $1; } + function cargo() { forklift cargo "$@" } fi # - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" From 4330b64fd6a18f9cf7a4cb35909380b1a358d164 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 18:19:20 +0300 Subject: [PATCH 12/37] ; --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 871c821a72d9..2c2fec698b77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,8 +126,8 @@ default: - export PATH=$PATH:$(pwd) - | if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo "$@" }" - function cargo() { forklift cargo "$@" } + echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo "$@"; }" + function cargo() { forklift cargo "$@"; } fi # - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" From eaec67674985be31d59f14eab301342a14249a34 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 7 Feb 2024 18:34:28 +0300 Subject: [PATCH 13/37] time --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 740a10d4472b..7cccee085f8f 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -316,7 +316,7 @@ build-linux-substrate: - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" - !reference [.forklift-cache, before_script] script: - - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p staging-node-cli + - time WASM_BUILD_NO_COLOR=1 cargo build --locked --release -p staging-node-cli - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then From 7a6f853dc436c0015d3b4faa24f6f79d49b29e52 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 13:12:40 +0300 Subject: [PATCH 14/37] script alias --- polkadot/scripts/build-only-wasm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index b6da3319c821..357a7a81dce3 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -6,6 +6,8 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` +shopt -s expand_aliases + if [ "$#" -lt 1 ]; then echo "You need to pass the name of the crate you want to compile!" exit 1 From 3b0c0591ecd27f376b91327e7437af5c7b5eb2eb Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 13:34:05 +0300 Subject: [PATCH 15/37] function --- polkadot/scripts/build-only-wasm.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index 357a7a81dce3..e32c78c00ba8 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -5,8 +5,13 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` - -shopt -s expand_aliases +function fl_cargo() { + if command -v forklift >/dev/null 2>&1; then + forklift cargo "$@"; + else + cargo "$@"; + fi +} if [ "$#" -lt 1 ]; then echo "You need to pass the name of the crate you want to compile!" @@ -24,8 +29,8 @@ fi if [ -d $WASM_BUILDER_RUNNER ]; then export DEBUG=false export OUT_DIR="$PROJECT_ROOT/target/release/build" - cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \ + fl_cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \ | grep -vE "cargo:rerun-if-|Executing build command" else - cargo build --release -p $1 + fl_cargo build --release -p $1 fi From 3fe4283a72fc6ec36012ce949e469f624fbd65ca Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 13:42:16 +0300 Subject: [PATCH 16/37] space --- polkadot/scripts/build-only-wasm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index e32c78c00ba8..d0a89efc6dd5 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -5,7 +5,7 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` -function fl_cargo() { +function fl_cargo () { if command -v forklift >/dev/null 2>&1; then forklift cargo "$@"; else From 1937dcb918a4704c07a96d7b29327f8b3cbdbe89 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 13:48:16 +0300 Subject: [PATCH 17/37] remove () --- polkadot/scripts/build-only-wasm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index d0a89efc6dd5..17ce0aa5c00a 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -5,7 +5,7 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` -function fl_cargo () { +function fl_cargo { if command -v forklift >/dev/null 2>&1; then forklift cargo "$@"; else From cc792f668318e4cf2f8714ebd51ee89fa66cd5dd Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 14:08:02 +0300 Subject: [PATCH 18/37] func syntax --- polkadot/scripts/build-only-wasm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index 17ce0aa5c00a..164d076dcdf0 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -5,7 +5,7 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` -function fl_cargo { +fl_cargo () { if command -v forklift >/dev/null 2>&1; then forklift cargo "$@"; else From 199e31212e446f065002449a5ee159a8d6413965 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Fri, 9 Feb 2024 14:32:36 +0300 Subject: [PATCH 19/37] if no forklift test --- polkadot/scripts/build-only-wasm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh index 164d076dcdf0..5c95b0082cd9 100755 --- a/polkadot/scripts/build-only-wasm.sh +++ b/polkadot/scripts/build-only-wasm.sh @@ -6,7 +6,7 @@ set -e PROJECT_ROOT=`git rev-parse --show-toplevel` fl_cargo () { - if command -v forklift >/dev/null 2>&1; then + if command -v forklift123 >/dev/null 2>&1; then forklift cargo "$@"; else cargo "$@"; From c63b98d9bae9b796cbeedb55f7d77f25744b1036 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 13 Feb 2024 17:52:35 +0300 Subject: [PATCH 20/37] compressors matrix --- .gitlab/pipeline/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 7cccee085f8f..41647a453d0c 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -71,6 +71,12 @@ build-malus: - .common-refs - .run-immediately - .collect-artifacts + palallel: + matrix: + - FORKLIFT_compression_type: lzma2 + FORKLIFT_compression_lzma2_compressionLevel: [1,2,3,4,5,6,7] + - FORKLIFT_compression_type: zstd + FORKLIFT_compression_zstd_compressionLevel: [1,2,3] script: - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts From 18b36981390ec1fea8eac115ae40191e6497336d Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 13 Feb 2024 19:00:00 +0300 Subject: [PATCH 21/37] none compressor --- .gitlab/pipeline/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 41647a453d0c..c2e73bf70d34 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -73,6 +73,7 @@ build-malus: - .collect-artifacts palallel: matrix: + - FORKLIFT_compression_type: none - FORKLIFT_compression_type: lzma2 FORKLIFT_compression_lzma2_compressionLevel: [1,2,3,4,5,6,7] - FORKLIFT_compression_type: zstd From 5af9c033741b9a90a262b32389601d591c8496f9 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 13 Feb 2024 19:09:52 +0300 Subject: [PATCH 22/37] typo --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 05418e1100f5..8bffe5d314dc 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -71,7 +71,7 @@ build-malus: - .common-refs - .run-immediately - .collect-artifacts - palallel: + parallel: matrix: - FORKLIFT_compression_type: none - FORKLIFT_compression_type: lzma2 From 79abfd77db10e82b5831720a8008bade660ff59e Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Mon, 19 Feb 2024 15:23:27 +0300 Subject: [PATCH 23/37] remove malus matrix --- .gitlab-ci.yml | 3 ++- .gitlab/pipeline/build.yml | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2ecfdaff811..68d2074f87e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,8 +124,9 @@ default: - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - export PATH=$PATH:$(pwd) + - export FORKLIFT_BYPASS=$FORKLIFT_BYPASS_ALT - | - if [ "$FORKLIFT_BYPASS_ALT" != "true" ]; then + if [ "$FORKLIFT_BYPASS" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating function cargo() { forklift cargo "$@"; }" function cargo() { forklift cargo "$@"; } fi diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 8bffe5d314dc..42aabc2635d4 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -71,13 +71,6 @@ build-malus: - .common-refs - .run-immediately - .collect-artifacts - parallel: - matrix: - - FORKLIFT_compression_type: none - - FORKLIFT_compression_type: lzma2 - FORKLIFT_compression_lzma2_compressionLevel: [1,2,3,4,5,6,7] - - FORKLIFT_compression_type: zstd - FORKLIFT_compression_zstd_compressionLevel: [1,2,3] script: - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts From 84ca8cecb4604b987e5315ebbeae1d276bda120b Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Sun, 7 Apr 2024 01:25:32 +0400 Subject: [PATCH 24/37] expand_aliases --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5b21dd021e8..c8baa38d3672 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,6 +127,7 @@ default: - > if [ "$FORKLIFT_BYPASS" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + shopt -s expand_aliases alias cargo="forklift cargo"; fi # From e7b2e316e8e112a9bf67903284d0bda3341f618f Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Sun, 7 Apr 2024 01:30:43 +0400 Subject: [PATCH 25/37] ; --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8baa38d3672..2b9bb6f6a8af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,7 +127,7 @@ default: - > if [ "$FORKLIFT_BYPASS" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - shopt -s expand_aliases + shopt -s expand_aliases; alias cargo="forklift cargo"; fi # From 1d0b7de495dd1ded873bd744a3f9298ae007916f Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 03:24:08 +0400 Subject: [PATCH 26/37] bundled forklift test --- .gitlab-ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b9bb6f6a8af..db451705ffa1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE] + CI_IMAGE: paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240408 # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master" @@ -119,19 +119,13 @@ default: # .forklift-cache: before_script: - - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' - - chmod +x forklift - - mkdir ~/.forklift - - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - - export PATH=$PATH:$(pwd) - > if [ "$FORKLIFT_BYPASS" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + forklift version; shopt -s expand_aliases; alias cargo="forklift cargo"; fi - # - - echo "FL_FORKLIFT_VERSION_ALT ${FL_FORKLIFT_VERSION_ALT}" .common-refs: From 45e5a5fe54ab8b3f64d5e789ada74b88dc1db4cc Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 03:39:06 +0400 Subject: [PATCH 27/37] return config --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db451705ffa1..4a1daed2dfe9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,13 +119,14 @@ default: # .forklift-cache: before_script: + - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - > if [ "$FORKLIFT_BYPASS" != "true" ]; then echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - forklift version; shopt -s expand_aliases; alias cargo="forklift cargo"; fi + # .common-refs: From 2440848065596c58884b4604a5f7c560931b9e2c Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 03:49:00 +0400 Subject: [PATCH 28/37] return .forklift/ --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a1daed2dfe9..2fb4ebc27691 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,7 @@ default: # .forklift-cache: before_script: + - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - > if [ "$FORKLIFT_BYPASS" != "true" ]; then From 66e0fe7b38c04ec1b23c1c15e92262be6dba3e51 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 05:20:14 +0400 Subject: [PATCH 29/37] images test --- .gitlab-ci.yml | 18 ++++++++++++++---- .gitlab/pipeline/build.yml | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fb4ebc27691..58dca5f089a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -116,17 +116,27 @@ default: tags: - linux-docker -# .forklift-cache: before_script: - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - > if [ "$FORKLIFT_BYPASS" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - shopt -s expand_aliases; - alias cargo="forklift cargo"; + echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + if command -v forklift >/dev/null 2>&1; then + echo "forklift already exist"; + else + echo "forklift does not exists, downloading"; + 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"'; + chmod +x forklift; + export PATH=$PATH:$(pwd); + fi + + shopt -s expand_aliases; + alias cargo="forklift cargo"; + fi + forklift version # diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 8658e92efc8f..ab433b64a857 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -17,6 +17,7 @@ build-linux-stable: RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" # Ensure we run the UI tests. RUN_UI_TESTS: 1 + CI_IMAGE: paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240222 script: - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ From 0c106e46ff6884dd0a90e33b6199f5d7977aea9b Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 05:26:33 +0400 Subject: [PATCH 30/37] fix --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58dca5f089a6..d90b3463b6bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,16 +127,13 @@ default: echo "forklift already exist"; else echo "forklift does not exists, downloading"; - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"'; + curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"; chmod +x forklift; export PATH=$PATH:$(pwd); fi - shopt -s expand_aliases; alias cargo="forklift cargo"; - fi - forklift version # From 3a85f4cbf7d0f51add138a353567f07cf9b6eaac Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 13:37:33 +0400 Subject: [PATCH 31/37] multi bin test --- .gitlab-ci.yml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d90b3463b6bc..fba3f734c068 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -116,25 +116,23 @@ default: tags: - linux-docker +# .forklift-cache: before_script: - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml + # + - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' + - chmod +x forklift + - export PATH=$PATH:$(pwd) - > if [ "$FORKLIFT_BYPASS" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - if command -v forklift >/dev/null 2>&1; then - echo "forklift already exist"; - else - echo "forklift does not exists, downloading"; - curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"; - chmod +x forklift; - export PATH=$PATH:$(pwd); - fi - shopt -s expand_aliases; - alias cargo="forklift cargo"; + echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + shopt -s expand_aliases; + alias cargo="forklift cargo"; fi # + - forklift version .common-refs: From 0354526ddebbd7f74df3d06a96533d52e5ff1ce4 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 13:56:55 +0400 Subject: [PATCH 32/37] echo --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fba3f734c068..2f4dcb6faeac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,6 +132,7 @@ default: alias cargo="forklift cargo"; fi # + - echo $PATH - forklift version From d14abe31da379ff38f58f072763f62b4309b0b7d Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 14:23:20 +0400 Subject: [PATCH 33/37] f --- .gitlab-ci.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f4dcb6faeac..402e1e4ad2fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -118,22 +118,25 @@ default: # .forklift-cache: - before_script: - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - # - - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"' - - chmod +x forklift - - export PATH=$PATH:$(pwd) - > if [ "$FORKLIFT_BYPASS" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - shopt -s expand_aliases; - alias cargo="forklift cargo"; + echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + if command -v forklift >/dev/null 2>&1; then + echo "forklift already exist"; + forklift version + else + echo "forklift does not exists, downloading"; + curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"; + chmod +x forklift; + export PATH=$PATH:$(pwd); + echo ${FL_FORKLIFT_VERSION_ALT} + fi + shopt -s expand_aliases; + alias cargo="forklift cargo"; fi # - - echo $PATH - - forklift version .common-refs: From 16534b569b14e48f50e8422f776772a8701cd48a Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 19:24:33 +0400 Subject: [PATCH 34/37] before_script --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 402e1e4ad2fe..849b8521921d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -118,6 +118,7 @@ default: # .forklift-cache: + before_script: - mkdir ~/.forklift - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml - > @@ -131,7 +132,7 @@ default: curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"; chmod +x forklift; export PATH=$PATH:$(pwd); - echo ${FL_FORKLIFT_VERSION_ALT} + echo ${FL_FORKLIFT_VERSION_ALT}; fi shopt -s expand_aliases; alias cargo="forklift cargo"; From 3c5746e259946df45c8d0f4883c409ca511a5e9a Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 9 Apr 2024 21:51:37 +0400 Subject: [PATCH 35/37] gcp img --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 849b8521921d..7a9d64af6418 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240408 + CI_IMAGE: europe-docker.pkg.dev/parity-build/ci-images/ci-unified:bullseye-1.75.0-2024-01-22-v20240408 # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master" From b0975d878fb683905ec44edc3f02fb6496c3482d Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 10 Apr 2024 04:06:04 +0400 Subject: [PATCH 36/37] old script test --- .gitlab-ci.yml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a9d64af6418..00b14cceec23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,24 +119,16 @@ default: # .forklift-cache: before_script: + - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"' + - chmod +x forklift - mkdir ~/.forklift - - cp $FL_FORKLIFT_CONFIG_ALT ~/.forklift/config.toml + - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml + - shopt -s expand_aliases + - export PATH=$PATH:$(pwd) - > if [ "$FORKLIFT_BYPASS" != "true" ]; then - echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; - if command -v forklift >/dev/null 2>&1; then - echo "forklift already exist"; - forklift version - else - echo "forklift does not exists, downloading"; - curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION_ALT}/forklift_${FL_FORKLIFT_VERSION_ALT}_linux_amd64"; - chmod +x forklift; - export PATH=$PATH:$(pwd); - echo ${FL_FORKLIFT_VERSION_ALT}; - fi - shopt -s expand_aliases; - alias cargo="forklift cargo"; - fi + echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; + alias cargo="forklift cargo"; # From bcb36fa75c538b4e6bde1162b8c3d528efce931f Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 15 May 2024 20:19:34 +0400 Subject: [PATCH 37/37] test image --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 031a0cc3fadb..7faf4249e755 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: europe-docker.pkg.dev/parity-build/ci-images/ci-unified:bullseye-1.75.0-2024-01-22-v20240408 + CI_IMAGE: europe-docker.pkg.dev/parity-build/ci-images/ci-unified:bullseye-1.77.0-2024-04-10-v20240515 # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master"