Merge branch 'master' into clean-up

.github/commands-readme.md

@@ -10,6 +10,7 @@ The current available command actions are:
 
 - [Command FMT](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-fmt.yml)
 - [Command Update UI](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-update-ui.yml)
+- [Command Prdoc](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-prdoc.yml)
 - [Command Sync](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-sync.yml)
 - [Command Bench](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench.yml)
 - [Command Bench All](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench-all.yml)
@@ -235,6 +236,16 @@ You can use the following [`gh cli`](https://cli.github.com/) inside the repo:
 gh workflow run command-bench-overheard.yml -f pr=1000 -f benchmark=substrate -f runtime=rococo -f target_dir=substrate
 ```
 
+### PrDoc
+
+Generate a PrDoc with the crates populated by all modified crates.
+
+Options:
+- `pr`: The PR number to generate the PrDoc for.
+- `audience`: The audience of whom the changes may concern.
+- `bump`: A default bump level for all crates. The PrDoc will likely need to be edited to reflect the actual changes after generation.
+- `overwrite`: Whether to overwrite any existing PrDoc.
+
 ### Sync
 
 Run sync and commit back results to PR.

.github/scripts/generate-prdoc.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""
+Generate the PrDoc for a Pull Request with a specific number, audience and bump level.
+
+It downloads and parses the patch from the GitHub API to opulate the prdoc with all modified crates.
+This will delete any prdoc that already exists for the PR if `--force` is passed.
+
+Usage:
+	python generate-prdoc.py --pr 1234 --audience "TODO" --bump "TODO"
+"""
+
+import argparse
+import os
+import re
+import sys
+import subprocess
+import toml
+import yaml
+import requests
+
+from github import Github
+import whatthepatch
+from cargo_workspace import Workspace
+
+# Download the patch and pass the info into `create_prdoc`.
+def from_pr_number(n, audience, bump, force):
+	print(f"Fetching PR '{n}' from GitHub")
+	g = Github()
+
+	repo = g.get_repo("paritytech/polkadot-sdk")
+	pr = repo.get_pull(n)
+
+	patch_url = pr.patch_url
+	patch = requests.get(patch_url).text
+
+	create_prdoc(n, audience, pr.title, pr.body, patch, bump, force)
+
+def create_prdoc(pr, audience, title, description, patch, bump, force):
+	path = f"prdoc/pr_{pr}.prdoc"
+
+	if os.path.exists(path):
+		if force == True:
+			print(f"Overwriting existing PrDoc for PR {pr}")
+		else:
+			print(f"PrDoc already exists for PR {pr}. Use --force to overwrite.")
+			sys.exit(1)
+	else:
+		print(f"No preexisting PrDoc for PR {pr}")
+
+	prdoc = { "doc": [{}], "crates": [] }
+
+	prdoc["title"] = title
+	prdoc["doc"][0]["audience"] = audience
+	prdoc["doc"][0]["description"] = description
+
+	workspace = Workspace.from_path(".")
+
+	modified_paths = []
+	for diff in whatthepatch.parse_patch(patch):
+		modified_paths.append(diff.header.new_path)
+
+	modified_crates = {}
+	for p in modified_paths:
+		# Go up until we find a Cargo.toml
+		p = os.path.join(workspace.path, p)
+		while not os.path.exists(os.path.join(p, "Cargo.toml")):
+			p = os.path.dirname(p)
+
+		with open(os.path.join(p, "Cargo.toml")) as f:
+			manifest = toml.load(f)
+
+		if not "package" in manifest:
+			print(f"File was not in any crate: {p}")
+			continue
+
+		crate_name = manifest["package"]["name"]
+		if workspace.crate_by_name(crate_name).publish:
+			modified_crates[crate_name] = True
+		else:
+			print(f"Skipping unpublished crate: {crate_name}")
+
+	print(f"Modified crates: {modified_crates.keys()}")
+
+	for crate_name in modified_crates.keys():
+		entry = { "name": crate_name }
+
+		if bump == 'silent' or bump == 'ignore' or bump == 'no change':
+			entry["validate"] = False
+		else:
+			entry["bump"] = bump
+
+		print(f"Adding crate {entry}")
+		prdoc["crates"].append(entry)
+
+	# write the parsed PR documentation back to the file
+	with open(path, "w") as f:
+		yaml.dump(prdoc, f)
+
+def parse_args():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--pr", type=int, required=True)
+	parser.add_argument("--audience", type=str, default="TODO")
+	parser.add_argument("--bump", type=str, default="TODO")
+	parser.add_argument("--force", type=str)
+	return parser.parse_args()
+
+if __name__ == "__main__":
+	args = parse_args()
+	force = True if args.force.lower() == "true" else False
+	print(f"Args: {args}, force: {force}")
+	from_pr_number(args.pr, args.audience, args.bump, force)

.github/workflows/command-prdoc.yml

@@ -0,0 +1,78 @@
+name: Command PrDoc
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        type: number
+        description: Number of the Pull Request
+        required: true
+      bump:
+        type: choice
+        description: Default bump level for all crates
+        default: "TODO"
+        required: true
+        options:
+          - "TODO"
+          - "no change"
+          - "patch"
+          - "minor"
+          - "major"
+      audience:
+        type: choice
+        description: Audience of the PrDoc
+        default: "TODO"
+        required: true
+        options:
+          - "TODO"
+          - "Runtime Dev"
+          - "Runtime User"
+          - "Node Dev"
+          - "Node User"
+      overwrite:
+        type: choice
+        description: Overwrite existing PrDoc
+        default: "true"
+        required: true
+        options:
+          - "true"
+          - "false"
+
+concurrency:
+  group: command-prdoc
+  cancel-in-progress: true
+
+jobs:
+  cmd-prdoc:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Download repo
+        uses: actions/checkout@v4
+      - name: Install gh cli
+        id: gh
+        uses: ./.github/actions/set-up-gh
+        with:
+          pr-number: ${{ inputs.pr }}
+          GH_TOKEN: ${{ github.token }}
+      - name: Generate PrDoc
+        run: |
+          python3 -m pip install -q cargo-workspace PyGithub whatthepatch pyyaml toml
+
+          python3 .github/scripts/generate-prdoc.py --pr "${{ inputs.pr }}" --bump "${{ inputs.bump }}" --audience "${{ inputs.audience }}" --force "${{ inputs.overwrite }}"
+
+      - name: Report failure
+        if: ${{ failure() }}
+        run: gh pr comment ${{ inputs.pr }} --body "<h2>Command failed ❌</h2> Run by @${{ github.actor }} for <code>${{ github.workflow }}</code> failed. See logs <a href=\"$RUN\">here</a>."
+        env:
+          RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GH_TOKEN: ${{ github.token }}
+      - name: Push Commit
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Add PrDoc (auto generated)
+          branch: ${{ steps.gh.outputs.branch }}
+          file_pattern: 'prdoc/*.prdoc'

.github/workflows/subsystem-benchmarks.yml

@@ -0,0 +1,82 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [ opened, synchronize, reopened, closed, labeled ]
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  set-image:
+    # TODO: remove once migration is complete or this workflow is fully stable
+    if: contains(github.event.label.name, 'GHA-migration')
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+
+  build:
+    needs: [ set-image ]
+    runs-on: arc-runners-polkadot-sdk-benchmark
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      BENCH_DIR: ./charts/bench/${{ matrix.features.bench }}
+      BENCH_FILE_NAME: ${{ matrix.features.bench }}
+    strategy:
+      fail-fast: false
+      matrix:
+        features: [
+          { name: "polkadot-availability-recovery", bench: "availability-recovery-regression-bench" },
+          { name: "polkadot-availability-distribution", bench: "availability-distribution-regression-bench" },
+          { name: "polkadot-node-core-approval-voting", bench: "approval-voting-regression-bench" },
+          { name: "polkadot-statement-distribution", bench: "statement-distribution-regression-bench" }
+        ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check Rust
+        run: |
+          rustup show
+          rustup +nightly show
+
+      - name: Run Benchmarks
+        continue-on-error: true
+        id: run-benchmarks
+        run: |
+          cargo bench -p ${{ matrix.features.name }} --bench ${{ matrix.features.bench }} --features subsystem-benchmarks || echo "Benchmarks failed"
+          ls -lsa ./charts
+          mkdir -p $BENCH_DIR || echo "Directory exists"
+          cp charts/${BENCH_FILE_NAME}.json $BENCH_DIR
+          ls -lsa $BENCH_DIR
+          # Fixes "detected dubious ownership" error in the ci
+          git config --global --add safe.directory '*'
+
+      - name: Publish result to GH Pages
+        if: ${{ steps.run-benchmarks.outcome == 'success' }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          name: ${{ env.BENCH_FILE_NAME }}
+          output-file-path: ${{ env.BENCH_DIR }}/${{ env.BENCH_FILE_NAME }}.json
+          benchmark-data-dir-path: ${{ env.BENCH_DIR }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: ${{ github.event_name == 'pull_request' }} # will comment on PRs if regression is detected
+          auto-push: false # TODO: enable when gitlab part is removed  ${{ github.ref == 'refs/heads/master' }}
+

Cargo.toml

@@ -819,7 +819,7 @@ lazy_static = { version = "1.4.0" }
 libc = { version = "0.2.155" }
 libfuzzer-sys = { version = "0.4" }
 libp2p = { version = "0.52.4" }
-libp2p-identity = { version = "0.2.3" }
+libp2p-identity = { version = "0.2.9" }
 libsecp256k1 = { version = "0.7.0", default-features = false }
 linked-hash-map = { version = "0.5.4" }
 linked_hash_set = { version = "0.1.4" }

cumulus/primitives/aura/Cargo.toml

@@ -10,24 +10,14 @@ description = "Core primitives for Aura in Cumulus"
 workspace = true
 
 [dependencies]
-codec = { features = ["derive"], workspace = true }
 
 # Substrate
 sp-api = { workspace = true }
 sp-consensus-aura = { workspace = true }
-sp-runtime = { workspace = true }
-
-# Polkadot
-polkadot-core-primitives = { workspace = true }
-polkadot-primitives = { workspace = true }
 
 [features]
 default = ["std"]
 std = [
-	"codec/std",
-	"polkadot-core-primitives/std",
-	"polkadot-primitives/std",
 	"sp-api/std",
 	"sp-consensus-aura/std",
-	"sp-runtime/std",
 ]