[skip ci] cmd.yml

remove rust installation + add forklift submodules + dev runtime adjst subweight update ui
paritytech-stg · Aug 22, 2024 · 76be7da · 76be7da
1 parent 85ddf4c
commit 76be7da
Show file tree

Hide file tree

Showing 4 changed files with 140 additions and 78 deletions.
diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -21,9 +21,11 @@
     '--quiet': {"action": "store_true", "help": "Won't print start/end/failed messages in Pull Request"},
     '--clean': {"action": "store_true", "help": "Clean up the previous bot's & author's comments in Pull Request "
                                                 "which triggered /cmd"},
+    '--image': {"action": "store_true", "help": "Full docker image to use for running the command. "
+                                                "i.e. --image=docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v202407161507"},
 }
 
-parser = argparse.ArgumentParser(prog="/cmd ", description='A command runner for polkadot runtimes repo', add_help=False)
+parser = argparse.ArgumentParser(prog="/cmd ", description='A command runner for polkadot-sdk repo', add_help=False)
 parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help')  # help for help
 
 subparsers = parser.add_subparsers(help='a command to run', dest='command')
@@ -34,22 +36,21 @@
 
 bench_example = '''**Examples**:
 
- > runs all benchmarks
+ - Runs all benchmarks
  
- %(prog)s
+ > %(prog)s
  
- > runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets
- > --quiet makes it to output nothing to PR but reactions
+ - Runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets. **--quiet** makes it to output nothing to PR but reactions
  
- %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet
- 
- > runs bench for all pallets for polkadot runtime and continues even if some benchmarks fail
- 
- %(prog)s --runtime polkadot --continue-on-fail 
+ > %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet
+
+ - Runs bench for all pallets for westend runtime and continues even if some benchmarks fail
  
- > does not output anything and cleans up the previous bot's & author command triggering comments in PR 
+ > %(prog)s --runtime westend --continue-on-fail
+
+ - Does not output anything and cleans up the previous bot's & author command triggering comments in PR 
  
- %(prog)s --runtime polkadot kusama --pallet pallet_balances pallet_multisig --quiet --clean 
+ > %(prog)s --runtime westend rococo --pallet pallet_balances pallet_multisig --quiet --clean
 
  '''
 
@@ -64,10 +65,18 @@
 """
 FMT 
 """
-parser_fmt = subparsers.add_parser('fmt', help='Formats code')
+parser_fmt = subparsers.add_parser('fmt', help='Formats code (cargo +nightly-VERSION fmt) and configs (taplo format)')
 for arg, config in common_args.items():
     parser_fmt.add_argument(arg, **config)
 
+"""
+Update UI 
+"""
+parser_ui = subparsers.add_parser('update-ui', help='Updates UI tests')
+for arg, config in common_args.items():
+    parser_ui.add_argument(arg, **config)
+
+
 args, unknown = parser.parse_known_args()
 
 print(f'args: {args}')
@@ -89,7 +98,7 @@
 
     # loop over remaining runtimes to collect available pallets
     for runtime in runtimesMatrix.values():
-        os.system(f"cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks")
+        os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks")
         print(f'-- listing pallets for benchmark for {runtime["name"]}')
         wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
         output = os.popen(
@@ -135,22 +144,23 @@
         for pallet in runtime_pallets_map[runtime]:
             config = runtimesMatrix[runtime]
             print(f'-- config: {config}')
-            default_path = f"./{config['path']}/src/weights"
-            xcm_path = f"./{config['path']}/src/weights/xcm"
-            output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path
+            if runtime == 'dev':
+                # to support sub-modules (https://github.com/paritytech/command-bot/issues/275)
+                search_manifest_path = f"cargo metadata --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .manifest_path'"
+                print(f'-- running: {search_manifest_path}')
+                manifest_path = os.popen(search_manifest_path).read()
+                package_dir = os.path.dirname(manifest_path)
+                print(f'-- package_dir: {package_dir}')
+                print(f'-- manifest_path: {manifest_path}')
+                output_path = os.path.join(package_dir, "src", "weights.rs")
+            else:
+                default_path = f"./{config['path']}/src/weights"
+                xcm_path = f"./{config['path']}/src/weights/xcm"
+                output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path
             print(f'-- benchmarking {pallet} in {runtime} into {output_path}')
-
-            status = os.system(f"frame-omni-bencher v1 benchmark pallet "
-                               f"--extrinsic=* "
-                               f"--runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm "
-                               f"--pallet={pallet} "
-                               f"--header={header_path} "
-                               f"--output={output_path} "
-                               f"--wasm-execution=compiled  "
-                               f"--steps=50 "
-                               f"--repeat=20 "
-                               f"--heap-pages=4096 "
-                               )
+            cmd = f"frame-omni-bencher v1 benchmark pallet --extrinsic=* --runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm --pallet={pallet} --header={header_path} --output={output_path} --wasm-execution=compiled --steps=50 --repeat=20 --heap-pages=4096"
+            print(f'-- Running: {cmd}')
+            status = os.system(cmd)
             if status != 0 and not args.continue_on_fail:
                 print(f'Failed to benchmark {pallet} in {runtime}')
                 sys.exit(1)
@@ -175,14 +185,30 @@
     tempdir.cleanup()
 
 elif args.command == 'fmt':
+    # Based on https://github.com/paritytech/scripts/blob/master/dockerfiles/ci-unified/Dockerfile
     nightly_version = os.getenv('RUST_NIGHTLY_VERSION')
     command = f"cargo +nightly-{nightly_version} fmt"
-    print('Formatting with `{command}`')
+    print(f'Formatting with `{command}`')
     nightly_status = os.system(f'{command}')
     taplo_status = os.system('taplo format --config .config/taplo.toml')
 
     if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail:
         print('❌ Failed to format code')
         sys.exit(1)
 
+elif args.command == 'update-ui':
+    command = '''
+        cargo test --manifest-path substrate/primitives/runtime-interface/Cargo.toml ui
+        cargo test -p sp-api-test ui
+        cargo test -p frame-election-provider-solution-type ui
+        cargo test -p frame-support-test --features=no-metadata-docs,try-runtime,experimental ui
+        cargo test -p xcm-procedural ui
+    '''
+    print(f'Updating ui with `{command}`')
+    status = os.system(f'{command}')
+
+    if status != 0 and not args.continue_on_fail:
+        print('❌ Failed to format code')
+        sys.exit(1)
+
 print('🚀 Done')
diff --git a/.github/workflows/cmd.yml b/.github/workflows/cmd.yml
@@ -12,28 +12,55 @@ permissions: # allow the action to comment on the PR
 
 jobs:
   is-org-member:
+    if: startsWith(github.event.comment.body, '/cmd')
     runs-on: ubuntu-latest
     outputs:
-      member: ${{ steps.is-member.outputs.is-member }}
+      member: ${{ steps.is-member.outputs.result }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Generate token
+        id: generate_token
+        uses: tibdex/github-app-token@v2.1.0
+        with:
+          app_id: ${{ secrets.CMD_BOT_APP_ID }}
+          private_key: ${{ secrets.CMD_BOT_APP_KEY }}
+
       - name: Check if user is a member of the organization
         id: is-member
         uses: actions/github-script@v7
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ steps.generate_token.outputs.token }}
+          result-encoding: string
           script: |
-            const membership = await github.orgs.checkMembershipForUser({
-              org: context.repo.owner,
-              username: context.actor
-            })
-            return membership.data.state === 'active'
+            const fs = require("fs");
+            try {              
+              const org = '${{ github.event.repository.owner.login }}';
+              const username = '${{ github.event.comment.user.login }}';
+            
+              const membership = await github.rest.orgs.checkMembershipForUser({
+                  org: org,
+                  username: username
+              });
+            
+              console.log(membership, membership.status, membership.status === 204);
+            
+              if (membership.status === 204) {
+                return 'true';
+              } else {
+                console.log(membership);
+                fs.appendFileSync(process.env["GITHUB_STEP_SUMMARY"], `${membership.data && membership.data.message || 'Unknown error happened, please check logs'}`);
+              }
+            } catch (error) {
+              console.log(error)
+            }
+            
+            return 'false';
 
-  reject-non-fellows:
+  reject-non-members:
     needs: is-org-member
-    if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(needs.fellows.outputs.github-handles, github.event.sender.login) }}
+    if: ${{ startsWith(github.event.comment.body, '/cmd') && needs.is-org-member.outputs.member != 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Add reaction to rejected comment
@@ -57,12 +84,12 @@ jobs:
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: `Sorry, only fellows can run commands.`
+              body: `Sorry, only ${{ github.event.repository.owner.login }} members can run commands.`
             })
 
   acknowledge:
-    needs: fellows
-    if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }}
+    needs: is-org-member
+    if: ${{ startsWith(github.event.comment.body, '/cmd') && needs.is-org-member.outputs.member == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Add reaction to triggered comment
@@ -85,7 +112,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Clean previous comments
-        if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--clean') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }}
+        if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--clean') && needs.is-org-member.outputs.member == 'true' }}
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -100,7 +127,13 @@ jobs:
                 if (
                   ${{ github.event.comment.id }} !== comment.id &&
                     (
-                      ((comment.body.startsWith('Command') || comment.body.startsWith('<details><summary>Command')) && comment.user.type === 'Bot') ||
+                      (
+                        (
+                          comment.body.startsWith('Command') || 
+                          comment.body.startsWith('<details><summary>Command') || 
+                          comment.body.startsWith('Sorry, only ')
+                        ) && comment.user.type === 'Bot'
+                      ) ||
                       (comment.body.startsWith('/cmd') && comment.user.login === context.actor)
                     )
                 ) {
@@ -114,7 +147,7 @@ jobs:
             })
   help:
     needs: [ clean, is-org-member ]
-    if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }}
+    if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--help') && needs.is-org-member.outputs.member == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -174,19 +207,34 @@ jobs:
               content: '+1'
             })
 
-  cmd:
+  set-image:
     needs: [ clean, is-org-member ]
-    env:
-      JOB_NAME: 'cmd'
-    if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }}
-    runs-on: ${{ startsWith(github.event.comment.body, '/cmd bench') && 'arc-runners-polkadot-sdk-benchmark' || 'ubuntu-latest' }}
+    if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(github.event.comment.body, '--help') && needs.is-org-member.outputs.member == 'true' }}
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set-image.outputs.IMAGE }}
     steps:
-      - name: Install updates and protobuf-compiler
-        if: startsWith(github.event.comment.body, '/cmd bench')
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - id: set-image
         run: |
-          sudo apt update && sudo apt install --assume-yes \
-            openssl pkg-config g++ make cmake protobuf-compiler curl libssl-dev libclang-dev libudev-dev git jq
+          IMAGE_OVERRIDE=$(echo "${{ github.event.comment.body }}" | grep -oe 'docker.io/paritytech/ci-unified:.*\s' | xargs)
+          
+          cat .github/env >> $GITHUB_OUTPUT
+          
+          if [ -n "$IMAGE_OVERRIDE" ]; then
+              echo "IMAGE=$IMAGE_OVERRIDE" >> $GITHUB_OUTPUT
+          fi
 
+  cmd:
+    needs: [ set-image ]
+    env:
+      JOB_NAME: 'cmd'
+    runs-on: ${{ startsWith(github.event.comment.body, '/cmd bench') && 'arc-runners-beefy-stg' || 'ubuntu-latest' }} # TODO: change to  arc-runners-polkadot-sdk-benchmark
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
       - name: Get command
         uses: actions-ecosystem/action-regex-match@v2
         id: get-pr-comment
@@ -235,30 +283,10 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Set rust version via common env file
-        run: cat .github/env >> $GITHUB_ENV
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
-        with:
-          targets: "wasm32-unknown-unknown,x86_64-unknown-linux-musl"
-          components: "rust-src rustfmt clippy"
-          toolchain: "nightly-${{env.RUST_NIGHTLY_VERSION}}"
-
       - name: Install dependencies for bench
         if: startsWith(steps.get-pr-comment.outputs.group2, 'bench')
         run: cargo install subweight frame-omni-bencher --locked
 
-      - name: Install dependencies for fmt
-        if: startsWith(steps.get-pr-comment.outputs.group2, 'fmt')
-        run: cargo install taplo-cli --version ${{ env.TAPLO_VERSION }}
-
-      - name: Setup Cache
-        if: startsWith(steps.get-pr-comment.outputs.group2, 'bench')
-        uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0
-        with:
-          shared-key: "fellowship-cmd"
-
       - name: Run cmd
         id: cmd
         env:
@@ -292,13 +320,13 @@ jobs:
         run: |
           git fetch
           result=$(subweight compare commits \
-            --path-pattern "./**/weights/**/*.rs" \
+            --path-pattern "./**/weights/**/*.rs,./**/weights.rs" \
             --method asymptotic \
             --format markdown \
             --no-color \
             --change added changed \
             --ignore-errors \
-            refs/remotes/origin/main ${{ github.ref }})
+            refs/remotes/origin/master ${{ github.ref }})
           
           # Save the multiline result to the output
           {

diff --git a/.github/workflows/runtimes-matrix.json b/.github/workflows/runtimes-matrix.json
@@ -1,4 +1,11 @@
 [
+  {
+    "name": "dev",
+    "package": "kitchensink-runtime",
+    "path": "substrate/frame",
+    "uri": null,
+    "is_relay": false
+  },
   {
     "name": "westend",
     "package": "westend-runtime",
@@ -45,7 +52,7 @@
     "name": "collectives-westend",
     "package": "collectives-westend-runtime",
     "path": "cumulus/parachains/runtimes/collectives/collectives-westend",
-    "uri": "wss://westend-collectives-rpc.polkadot.io:443",
+    "uri": "wss://westend-collectives-rpc.polkadot.io:443"
   },
   {
     "name": "contracts-rococo",

diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@
 **/node_modules
 **/target/
 **/wip/*.stderr
+**/__pycache__/
 /.cargo/config
 /.envrc
 artifacts