Merge remote-tracking branch 'upstream/master' into syncing-strategy-…

…refactoring-part-3

# Conflicts:
#	substrate/client/network/sync/src/engine.rs

.cargo/config.toml

@@ -8,3 +8,4 @@ rustdocflags = [
 # Needed for musl builds so user doesn't have to install musl-tools.
 CC_x86_64_unknown_linux_musl = { value = ".cargo/musl-gcc", force = true, relative = true }
 CXX_x86_64_unknown_linux_musl = { value = ".cargo/musl-g++", force = true, relative = true }
+CARGO_WORKSPACE_ROOT_DIR = { value = "", relative = true }

.github/scripts/deny-git-deps.py

@@ -15,6 +15,7 @@
 	'simple-mermaid': ['xcm-docs'],
 	# Fix in <https://github.com/paritytech/polkadot-sdk/issues/2922>
 	'bandersnatch_vrfs': ['sp-core'],
+	'subwasmlib': ['polkadot-zombienet-sdk-tests'],
 }
 
 root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
@@ -24,7 +25,7 @@
 def check_dep(dep, used_by):
 	if dep.location != DependencyLocation.GIT:
 		return
-	
+
 	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
 		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
 	else:

.github/workflows/tests-misc.yml

@@ -14,7 +14,6 @@ concurrency:
 # Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
@@ -321,7 +320,7 @@ jobs:
 
   cargo-check-all-crate-macos:
     timeout-minutes: 30
-    needs: [ preflight ]
+    needs: [preflight]
     runs-on: parity-macos
     env:
       SKIP_WASM_BUILD: 1
@@ -366,6 +365,7 @@ jobs:
       - check-tracing
       - cargo-check-each-crate
       - test-deterministic-wasm
+      - cargo-check-all-crate-macos
       # - cargo-hfuzz remove from required for now, as it's flaky
     if: always() && !cancelled()
     steps:

.gitlab-ci.yml

@@ -21,7 +21,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
-  CI_IMAGE: !reference [ .ci-unified, variables, CI_IMAGE ]
+  CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE]
   # BUILDAH_IMAGE is defined in group variables
   BUILDAH_COMMAND: "buildah --storage-driver overlay2"
   RELENG_SCRIPTS_BRANCH: "master"
@@ -39,7 +39,7 @@ default:
       - runner_system_failure
       - unknown_failure
       - api_failure
-  cache: { }
+  cache: {}
   interruptible: true
 
 .collect-artifacts:
@@ -68,8 +68,8 @@ default:
 
 .common-before-script:
   before_script:
-    - !reference [ .job-switcher, before_script ]
-    - !reference [ .pipeline-stopper-vars, script ]
+    - !reference [.job-switcher, before_script]
+    - !reference [.pipeline-stopper-vars, script]
 
 .job-switcher:
   before_script:
@@ -78,8 +78,8 @@ default:
 .kubernetes-env:
   image: "${CI_IMAGE}"
   before_script:
-    - !reference [ .common-before-script, before_script ]
-    - !reference [ .prepare-env, before_script ]
+    - !reference [.common-before-script, before_script]
+    - !reference [.prepare-env, before_script]
   tags:
     - kubernetes-parity-build
 
@@ -107,12 +107,12 @@ default:
 .docker-env:
   image: "${CI_IMAGE}"
   variables:
-    FL_FORKLIFT_VERSION: !reference [ .forklift, variables, FL_FORKLIFT_VERSION ]
+    FL_FORKLIFT_VERSION: !reference [.forklift, variables, FL_FORKLIFT_VERSION]
   before_script:
-    - !reference [ .common-before-script, before_script ]
-    - !reference [ .prepare-env, before_script ]
-    - !reference [ .rust-info-script, script ]
-    - !reference [ .forklift-cache, before_script ]
+    - !reference [.common-before-script, before_script]
+    - !reference [.prepare-env, before_script]
+    - !reference [.rust-info-script, script]
+    - !reference [.forklift-cache, before_script]
   tags:
     - linux-docker
 
@@ -288,8 +288,3 @@ cancel-pipeline-build-short-benchmark:
   extends: .cancel-pipeline-template
   needs:
     - job: build-short-benchmark
-
-cancel-pipeline-cargo-check-each-crate-macos:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: cargo-check-each-crate-macos

.gitlab/pipeline/build.yml

@@ -173,6 +173,25 @@ build-short-benchmark:
     - target/release/polkadot --version
     - cp ./target/release/polkadot ./artifacts/
 
+build-polkadot-zombienet-tests:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  needs:
+    - job: build-linux-stable
+      artifacts: true
+    - job: build-linux-stable-cumulus
+      artifacts: true
+
+  script:
+    - cargo nextest --manifest-path polkadot/zombienet-sdk-tests/Cargo.toml archive --features zombie-metadata --archive-file polkadot-zombienet-tests.tar.zst
+    - mkdir -p artifacts
+    - cp polkadot-zombienet-tests.tar.zst ./artifacts
+
+
 # build jobs from cumulus
 
 build-linux-stable-cumulus:

.gitlab/pipeline/test.yml

@@ -153,25 +153,3 @@ quick-benchmarks-omni:
   script:
     - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
     - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
-
-cargo-check-each-crate-macos:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    # - .collect-artifacts
-  before_script:
-    # skip timestamp script, the osx bash doesn't support printf %()T
-    - !reference [.job-switcher, before_script]
-    - !reference [.rust-info-script, script]
-    - !reference [.pipeline-stopper-vars, script]
-  variables:
-    SKIP_WASM_BUILD: 1
-  script:
-    # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
-    # - time ./scripts/ci/gitlab/check-each-crate.py 1 1
-    - time cargo check --workspace --locked
-  timeout: 2h
-  tags:
-    - osx

.gitlab/pipeline/zombienet.yml

@@ -4,6 +4,10 @@
     ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.105"
     PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics"
     DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs"
+    ZOMBIE_PROVIDER: "k8s"
+    RUST_LOG: "info,zombienet_orchestrator=debug"
+    RUN_IN_CI: "1"
+  timeout: 60m
 
 include:
   # substrate tests

.gitlab/pipeline/zombienet/parachain-template.yml

@@ -17,6 +17,8 @@
     RUST_LOG: "info,zombienet_orchestrator=debug"
     FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
     RUN_IN_CONTAINER: "1"
+    RUNNER_SCRIPT_TIMEOUT: 15m
+    RUNNER_AFTER_SCRIPT_TIMEOUT: 5m
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
     when: always
@@ -25,9 +27,9 @@
       - ./zombienet-logs
   after_script:
     - mkdir -p ./zombienet-logs
-    - cp /tmp/zombie*/logs/* ./zombienet-logs/
+    - cp /tmp/zombie*/*/*.log ./zombienet-logs/
   retry: 2
-  timeout: 15m
+  timeout: 20m
   tags:
     - linux-docker
 

.gitlab/pipeline/zombienet/polkadot.yml

@@ -31,6 +31,12 @@
     - echo "colander image ${COL_IMAGE}"
     - echo "cumulus image ${CUMULUS_IMAGE}"
     - echo "malus image ${MALUS_IMAGE}"
+    # RUN_IN_CONTAINER is env var that is set in the dockerfile
+    - if  [[ -v RUN_IN_CONTAINER  ]]; then
+      echo "Initializing zombie cluster";
+      gcloud auth activate-service-account --key-file "/etc/zombie-net/sa-zombie.json";
+      gcloud container clusters get-credentials parity-zombienet --zone europe-west3-b --project parity-zombienet;
+      fi
   stage: zombienet
   image: "${ZOMBIENET_IMAGE}"
   needs:
@@ -54,6 +60,7 @@
     MALUS_IMAGE: "docker.io/paritypr/malus"
     GH_DIR: "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
     LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/polkadot/zombienet_tests"
+    LOCAL_SDK_TEST: "/builds/parity/mirrors/polkadot-sdk/polkadot/zombienet-sdk-tests"
     FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
     RUN_IN_CONTAINER: "1"
   artifacts:
@@ -335,3 +342,18 @@ zombienet-polkadot-malus-0001-dispute-valid:
     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
       --local-dir="${LOCAL_DIR}/integrationtests"
       --test="0001-dispute-valid-block.zndsl"
+
+zombienet-polkadot-coretime-revenue:
+  extends:
+    - .zombienet-polkadot-common
+  needs:
+    - job: build-polkadot-zombienet-tests
+      artifacts: true
+  before_script:
+    - !reference [".zombienet-polkadot-common", "before_script"]
+    - export POLKADOT_IMAGE="${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  script:
+  # we want to use `--no-capture` in zombienet tests.
+    - unset NEXTEST_FAILURE_OUTPUT
+    - unset NEXTEST_SUCCESS_OUTPUT
+    - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- smoke::coretime_revenue::coretime_revenue_test