diff --git a/.github/env b/.github/env index 17e6bea555..6ff0ec8a9b 100644 --- a/.github/env +++ b/.github/env @@ -1,2 +1,3 @@ RUST_STABLE_VERSION=1.77.0 RUST_NIGHTLY_VERSION=2024-04-14 +TAPLO_VERSION=0.8.1 diff --git a/.github/scripts/cmd/_help.py b/.github/scripts/cmd/_help.py new file mode 100644 index 0000000000..8ad49dad84 --- /dev/null +++ b/.github/scripts/cmd/_help.py @@ -0,0 +1,26 @@ +import argparse + +""" + +Custom help action for argparse, it prints the help message for the main parser and all subparsers. + +""" + + +class _HelpAction(argparse._HelpAction): + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + + # retrieve subparsers from parser + subparsers_actions = [ + action for action in parser._actions + if isinstance(action, argparse._SubParsersAction)] + # there will probably only be one subparser_action, + # but better save than sorry + for subparsers_action in subparsers_actions: + # get all subparsers and print help + for choice, subparser in subparsers_action.choices.items(): + print("\n### Command '{}'".format(choice)) + print(subparser.format_help()) + + parser.exit() diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py new file mode 100755 index 0000000000..589ce7e173 --- /dev/null +++ b/.github/scripts/cmd/cmd.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +import os +import sys +import json +import argparse +import tempfile +import _help + +_HelpAction = _help._HelpAction + +f = open('.github/workflows/runtimes-matrix.json', 'r') +runtimesMatrix = json.load(f) + +runtimeNames = list(map(lambda x: x['name'], runtimesMatrix)) + +common_args = { + '--continue-on-fail': {"action": "store_true", "help": "Won't exit(1) on failed command and continue with next " + "steps. Helpful when you want to push at least successful " + "pallets, and then run failed ones separately"}, + '--quiet': {"action": "store_true", "help": "Won't print start/end/failed messages in Pull Request"}, + '--clean': {"action": "store_true", "help": "Clean up the previous bot's & author's comments in Pull Request " + "which triggered /cmd"}, +} + +parser = argparse.ArgumentParser(prog="/cmd ", description='A command runner for polkadot runtimes repo', add_help=False) +parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help') # help for help + +subparsers = parser.add_subparsers(help='a command to run', dest='command') + +""" +BENCH +""" + +bench_example = '''**Examples**: + + > runs all benchmarks + + %(prog)s + + > runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets + > --quiet makes it to output nothing to PR but reactions + + %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet + + > runs bench for all pallets for polkadot runtime and continues even if some benchmarks fail + + %(prog)s --runtime polkadot --continue-on-fail + + > does not output anything and cleans up the previous bot's & author command triggering comments in PR + + %(prog)s --runtime polkadot kusama --pallet pallet_balances pallet_multisig --quiet --clean + + ''' + +parser_bench = subparsers.add_parser('bench', help='Runs benchmarks', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter) + +for arg, config in common_args.items(): + parser_bench.add_argument(arg, **config) + +parser_bench.add_argument('--runtime', help='Runtime(s) space separated', choices=runtimeNames, nargs='*', default=runtimeNames) +parser_bench.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[]) + +""" +FMT +""" +parser_fmt = subparsers.add_parser('fmt', help='Formats code') +for arg, config in common_args.items(): + parser_fmt.add_argument(arg, **config) + +args, unknown = parser.parse_known_args() + +print(f'args: {args}') + +if args.command == 'bench': + tempdir = tempfile.TemporaryDirectory() + print(f'Created temp dir: {tempdir.name}') + runtime_pallets_map = {} + failed_benchmarks = {} + successful_benchmarks = {} + + profile = "release" + + print(f'Provided runtimes: {args.runtime}') + # convert to mapped dict + runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix)) + runtimesMatrix = {x['name']: x for x in runtimesMatrix} + print(f'Filtered out runtimes: {runtimesMatrix}') + + # loop over remaining runtimes to collect available pallets + for runtime in runtimesMatrix.values(): + os.system(f"cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks") + print(f'-- listing pallets for benchmark for {runtime["name"]}') + wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm" + output = os.popen( + f"frame-omni-bencher v1 benchmark pallet --no-csv-header --all --list --runtime={wasm_file}").read() + raw_pallets = output.split('\n') + + all_pallets = set() + for pallet in raw_pallets: + if pallet: + all_pallets.add(pallet.split(',')[0].strip()) + + pallets = list(all_pallets) + print(f'Pallets in {runtime}: {pallets}') + runtime_pallets_map[runtime['name']] = pallets + + # filter out only the specified pallets from collected runtimes/pallets + if args.pallet: + print(f'Pallet: {args.pallet}') + new_pallets_map = {} + # keep only specified pallets if they exist in the runtime + for runtime in runtime_pallets_map: + if set(args.pallet).issubset(set(runtime_pallets_map[runtime])): + new_pallets_map[runtime] = args.pallet + + runtime_pallets_map = new_pallets_map + + print(f'Filtered out runtimes & pallets: {runtime_pallets_map}') + + if not runtime_pallets_map: + if args.pallet and not args.runtime: + print(f"No pallets [{args.pallet}] found in any runtime") + elif args.runtime and not args.pallet: + print(f"{args.runtime} runtime does not have any pallets") + elif args.runtime and args.pallet: + print(f"No pallets [{args.pallet}] found in {args.runtime}") + else: + print('No runtimes found') + sys.exit(0) + + header_path = os.path.abspath('./.github/scripts/cmd/file_header.txt') + + for runtime in runtime_pallets_map: + for pallet in runtime_pallets_map[runtime]: + config = runtimesMatrix[runtime] + print(f'-- config: {config}') + default_path = f"./{config['path']}/src/weights" + xcm_path = f"./{config['path']}/src/weights/xcm" + output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path + print(f'-- benchmarking {pallet} in {runtime} into {output_path}') + + status = os.system(f"frame-omni-bencher v1 benchmark pallet " + f"--extrinsic=* " + f"--runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm " + f"--pallet={pallet} " + f"--header={header_path} " + f"--output={output_path} " + f"--wasm-execution=compiled " + f"--steps=50 " + f"--repeat=20 " + f"--heap-pages=4096 " + ) + if status != 0 and not args.continue_on_fail: + print(f'Failed to benchmark {pallet} in {runtime}') + sys.exit(1) + + # Otherwise collect failed benchmarks and print them at the end + # push failed pallets to failed_benchmarks + if status != 0: + failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet] + else: + successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet] + + if failed_benchmarks: + print('❌ Failed benchmarks of runtimes/pallets:') + for runtime, pallets in failed_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + if successful_benchmarks: + print('βœ… Successful benchmarks of runtimes/pallets:') + for runtime, pallets in successful_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + tempdir.cleanup() + +elif args.command == 'fmt': + nightly_version = os.getenv('RUST_NIGHTLY_VERSION') + command = f"cargo +nightly-{nightly_version} fmt" + print('Formatting with `{command}`') + nightly_status = os.system(f'{command}') + taplo_status = os.system('taplo format --config .config/taplo.toml') + + if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail: + print('❌ Failed to format code') + sys.exit(1) + +print('πŸš€ Done') diff --git a/.github/scripts/cmd/file_header.txt b/.github/scripts/cmd/file_header.txt new file mode 100644 index 0000000000..9fc718fdc0 --- /dev/null +++ b/.github/scripts/cmd/file_header.txt @@ -0,0 +1,15 @@ +// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md +// for a list of specific contributors. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/.github/workflows/cmd.yml b/.github/workflows/cmd.yml new file mode 100644 index 0000000000..10c82e0650 --- /dev/null +++ b/.github/workflows/cmd.yml @@ -0,0 +1,364 @@ +name: Command + +on: + issue_comment: # listen for comments on issues + types: [created] + +permissions: # allow the action to comment on the PR + contents: write + issues: write + pull-requests: write + actions: read + +jobs: + fellows: + runs-on: ubuntu-latest + outputs: + github-handles: ${{ steps.load-fellows.outputs.github-handles }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load fellows + id: load-fellows + uses: paritytech/get-fellows-action@v1.2.0 + + reject-non-fellows: + needs: fellows + if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Add reaction to rejected comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Comment PR (Rejected) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Sorry, only fellows can run commands.` + }) + + acknowledge: + needs: fellows + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Add reaction to triggered comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'eyes' + }) + + clean: + needs: fellows + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clean previous comments + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--clean') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(comments => { + for (let comment of comments.data) { + console.log(comment) + if ( + ${{ github.event.comment.id }} !== comment.id && + ( + ((comment.body.startsWith('Command') || comment.body.startsWith('
Command')) && comment.user.type === 'Bot') || + (comment.body.startsWith('/cmd') && comment.user.login === context.actor) + ) + ) { + github.rest.issues.deleteComment({ + comment_id: comment.id, + owner: context.repo.owner, + repo: context.repo.repo + }) + } + } + }) + help: + needs: [clean, fellows] + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )([\s\w-]+)$' + + - name: Save output of help + id: help + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo 'help<> $GITHUB_OUTPUT + python3 .github/scripts/cmd/cmd.py $CMD >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Comment PR (Help) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `
Command help:${{ steps.help.outputs.help }}
` + }) + + - name: Add confused reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + cmd: + needs: [clean, fellows] + env: + JOB_NAME: 'cmd' + if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ${{ startsWith(github.event.comment.body, '/cmd bench') && 'gitrun-001' || 'ubuntu-22.04' }} + steps: + - name: Install updates and protobuf-compiler + if: startsWith(github.event.comment.body, '/cmd bench') + run: | + sudo apt update && sudo apt install --assume-yes \ + openssl pkg-config g++ make cmake protobuf-compiler curl libssl-dev libclang-dev libudev-dev git jq + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )([\s\w-]+)$' + + - name: Build workflow link + if: ${{ !contains(github.event.comment.body, '--quiet') }} + id: build-link + run: | + # Get exactly the CMD job link, filtering out the other jobs + jobLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs | jq '.jobs[] | select(.name | contains("${{ env.JOB_NAME }}")) | .html_url') + + runLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} | jq '.html_url') + + echo "job_url=${jobLink}" + echo "run_url=${runLink}" + echo "job_url=$jobLink" >> $GITHUB_OUTPUT + echo "run_url=$runLink" >> $GITHUB_OUTPUT + + + - name: Comment PR (Start) + if: ${{ !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let job_url = ${{ steps.build-link.outputs.job_url }} + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has started πŸš€ [See logs here](${job_url})` + }) + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Set rust version via common env file + run: cat .github/env >> $GITHUB_ENV + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + targets: "wasm32-unknown-unknown,x86_64-unknown-linux-musl" + components: "rust-src rustfmt clippy" + toolchain: "nightly-${{env.RUST_NIGHTLY_VERSION}}" + + - name: Install dependencies for bench + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + run: cargo install subweight frame-omni-bencher --locked + + - name: Install dependencies for fmt + if: startsWith(steps.get-pr-comment.outputs.group2, 'fmt') + run: cargo install taplo-cli --version ${{ env.TAPLO_VERSION }} + + - name: Setup Cache + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0 + with: + shared-key: "fellowship-cmd" + + - name: Run cmd + id: cmd + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo "Running command: $CMD" + git remote -v + python3 .github/scripts/cmd/cmd.py $CMD + git status + git diff + + - name: Commit changes + run: | + if [ -n "$(git status --porcelain)" ]; then + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git pull origin ${{ github.head_ref }} + git add . + git restore --staged Cargo.lock # ignore changes in Cargo.lock + git commit -m "Update from ${{ github.actor }} running command '${{ steps.get-pr-comment.outputs.group2 }}'" || true + git push origin ${{ github.head_ref }} + else + echo "Nothing to commit"; + fi + + - name: Run Subweight + id: subweight + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + shell: bash + run: | + git fetch + result=$(subweight compare commits \ + --path-pattern "./**/weights/**/*.rs" \ + --method asymptotic \ + --format markdown \ + --no-color \ + --change added changed \ + --ignore-errors \ + refs/remotes/origin/main ${{ github.ref }}) + + # Save the multiline result to the output + { + echo "result<> $GITHUB_OUTPUT + + - name: Comment PR (End) + if: ${{ !failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + env: + SUBWEIGHT: '${{ steps.subweight.outputs.result }}' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let runUrl = ${{ steps.build-link.outputs.run_url }} + let subweight = process.env.SUBWEIGHT; + + let subweightCollapsed = subweight + ? `
\n\nSubweight results:\n\n${subweight}\n\n
` + : ''; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has finished βœ… [See logs here](${runUrl})${subweightCollapsed}` + }) + + - name: Comment PR (Failure) + if: ${{ failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let jobUrl = ${{ steps.build-link.outputs.job_url }} + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has failed ❌! [See logs here](${jobUrl})` + }) + + - name: Add πŸ˜• reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml index 11d4df7372..2b115bd863 100644 --- a/.github/workflows/fmt.yml +++ b/.github/workflows/fmt.yml @@ -24,7 +24,12 @@ jobs: run: cat .github/env >> $GITHUB_ENV - name: Install nightly toolchain - run: rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + run: | + rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + cargo install taplo-cli --version $TAPLO_VERSION - name: Rustfmt (check) run: cargo +nightly-$RUST_NIGHTLY_VERSION fmt --all -- --check + + - name: Taplo (check) + run: taplo format --check --config .config/taplo.toml diff --git a/.gitignore b/.gitignore index 4c9006e4e5..d591ea0a06 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ rls*.log runtime/wasm/target/ substrate.code-workspace target/ +**/__pycache__/ diff --git a/docs/running-commands.md b/docs/running-commands.md new file mode 100644 index 0000000000..a7d55e371c --- /dev/null +++ b/docs/running-commands.md @@ -0,0 +1,32 @@ +# Running Commands in PRs + +You can run commands in PRs by triggering it via comment. It will use the context of your PR and post the results back. + +## Usage + +`/cmd --help` to see all available commands and usage format + +`/cmd --help` to see the usage of a specific command + + +### Commands + +- `/cmd fmt` to format the code in the PR. It commits back with the formatted code (fmt) and configs (taplo). + +- `/cmd bench` to generate weights for a runtime. Read more about [Weight Generation](./weight-generation.md) + +### Flags + +1.`--quiet` to suppress the output of the command in the comments. +By default, the Start and End/Failure of the command will be commented with the link to a pipeline. +If you want to avoid, use this flag. Go to [Action Tab](https://github.com/polkadot-fellows/runtimes/actions/workflows/cmd.yml) to see the pipeline status. + +2.`--continue-on-fail` to continue running the command even if something inside a command (like specific pallet weight generation) are failed. +Basically avoids interruption in the middle with `exit 1` +The pipeline logs will include what is failed (like which runtimes/pallets), then you can re-run them separately or not. + +3.`--clean` to clean up all yours and bot's comments in PR relevant to `/cmd` commands. If you run too many commands, or they keep failing, and you're rerunning them again, it's handy to add this flag to keep a PR clean. + +### Adding new Commands +Feel free to add new commands to the workflow, however **_note_** that triggered workflows will use the actions from `main` (default) branch, meaning they will take effect only after the PR with new changes/command is merged. +If you want to test the new command, it's better to test in your fork and local-to-fork PRs, where you control the default branch. diff --git a/docs/weight-generation.md b/docs/weight-generation.md index 6904db0ca3..fea2d41f5f 100644 --- a/docs/weight-generation.md +++ b/docs/weight-generation.md @@ -1,101 +1,43 @@ # Weight Generation -To generate weights for a runtime +To generate weights for a runtime. +Weights generation is using self-hosted runner which is provided by [Amforc](https://amforc.com/), the rest commands are using standard Github runners on `ubuntu-latest` or `ubuntu-20.04`. +Self-hosted runner for benchmarks is configured to meet requirements of reference hardware for running validators https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware -1. Build `chain-spec-generator` with `--profile production --features runtime-benchmarks` -2. Use it to build a chain spec for your runtime, e.g. `./target/production/chain-spec-generator --raw polkadot-local > polkadot-chain-spec.json` -3. Create `file_header.txt` +In a PR run the actions through comment: -```text -// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md -// for a list of specific contributors. -// SPDX-License-Identifier: Apache-2.0 +```sh +/cmd bench --help # outputs the actual usage documentation with examples and supported runtimes -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -``` - -4. `rsync` chain spec/s and the file header to a benchmark machine - -5. Build `polkadot` binary from the latest release of `polkadot-sdk` with `--profile production --features runtime-benchmarks --bin polkadot` on the benchmark machine - -6. Run on the benchmark machine: - -```bash -#!/bin/bash - -# Default value is 'polkadot', but you can override it by passing a different value as an argument -CHAIN=${1:-polkadot} - -pallets=($( - ./target/production/polkadot benchmark pallet --list \ - --chain=./$CHAIN-chain-spec.json | - tail -n+2 | - cut -d',' -f1 | - sort | - uniq -)); +# or -mkdir -p ./$CHAIN-weights -for pallet in "${pallets[@]}"; do - output_file=./$CHAIN-weights/ - # a little hack for pallet_xcm_benchmarks - we want to output them to a nested directory - if [[ "$pallet" == "pallet_xcm_benchmarks::generic" ]] || [[ "$pallet" == "pallet_xcm_benchmarks::fungible" ]]; then - mkdir -p ./$CHAIN-weights/xcm - output_file="${output_file}xcm/${pallet//::/_}.rs" - fi - echo "Running benchmarks for $pallet to $output_file" - ./target/production/polkadot benchmark pallet \ - --chain=./$CHAIN-chain-spec.json \ - --steps=50 \ - --repeat=20 \ - --pallet=$pallet \ - --extrinsic=* \ - --wasm-execution=compiled \ - --heap-pages=4096 \ - --output="$output_file" \ - --header=./file_header.txt -done +/cmd --help # to see all available commands ``` -You probably want to do this inside a `tmux` session or something similar (e.g., `nohup &`), as it will take a while (several hours). - -7. `rsync` the weights back to your local machine, replacing the existing weights. - -8. Manually fix XCM weights by -- Replacing `impl xxx::yyy::WeightInfo for WeightInfo {` with `impl WeightInfo {` -- Marking all functions `pub(crate)` -- Removing any unused functions - -9. Commit the weight changes. +To generate weights for all pallets in a particular runtime(s), run the following command: +```sh +/cmd bench --runtime kusama polkadot +``` -10. Ensure the changes are reasonable. If not installed, `cargo install subweight`, check the weight changes: - ``` - subweight compare commits \ - --path-pattern "./**/weights/**/*.rs" \ - --method asymptotic \ - --ignore-errors \ - \ - ` - ``` - _Hint1: Add `--format markdown --no-color` for markdown-compatible results._ +> **πŸ“ Note**: The action is not being run right-away, it will be queued and run in the next available runner. So might be quick, but might also take up to 10 mins (That's in control of Github). +Once the action is run, you'll see reaction πŸ‘€ on original comment, and if you didn't pass `--quiet` - it will also send a link to a pipeline when started, and link to whole workflow when finished. - _Hint2: Change `--path-pattern "./**/weights/**/*.rs"` to e.g. `--path-pattern "./relay/polkadot/weights/**/*.rs"` for a specific runtime._ +> **πŸ’‘Hint #1** : if you run all runtimes or all pallets, it might be that some pallet in the middle is failed to generate weights, thus it stops (fails) the whole pipeline. +> If you want, you can make it to continue running, even if some pallets are failed, add `--continue-on-fail` flag to the command. The report will include which runtimes/pallets have failed, then you can re-run them separately after all is done. - _Hint3: Add `--change added changed` to include only relevant changes._ +This way it runs all possible runtimes for the specified pallets, if it finds them in the runtime +```sh +/cmd bench --pallet pallet_balances pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -## FAQ +If you want to run all specific pallet(s) for specific runtime(s), you can do it like this: +```sh +/cmd bench --runtime bridge-hub-polkadot --pallet pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -### What benchmark machine spec should I use? -See the [Polkadot Wiki Reference Hardware](https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#standard-hardware). +> **πŸ’‘Hint #2** : Sometimes when you run too many commands, or they keep failing and you're rerunning them again, it's handy to add `--clean` flag to the command. This will clean up all yours and bot's comments in PR relevant to /cmd commands. +```sh +/cmd bench --runtime kusama polkadot --pallet=pallet_balances --clean --continue-on-fail +``` diff --git a/relay/common/Cargo.toml b/relay/common/Cargo.toml index 1b6ded352a..0b735c6645 100644 --- a/relay/common/Cargo.toml +++ b/relay/common/Cargo.toml @@ -11,10 +11,10 @@ version.workspace = true codec = { features = ["derive", "max-encoded-len"], workspace = true } scale-info = { features = ["derive"], workspace = true } -sp-api ={ workspace = true } +sp-api = { workspace = true } sp-runtime = { workspace = true } polkadot-primitives = { workspace = true } -pallet-staking-reward-fn ={ workspace = true } +pallet-staking-reward-fn = { workspace = true } [features] @@ -23,8 +23,8 @@ std = [ "codec/std", "scale-info/std", + "pallet-staking-reward-fn/std", + "polkadot-primitives/std", "sp-api/std", "sp-runtime/std", - "polkadot-primitives/std", - "pallet-staking-reward-fn/std", ] diff --git a/relay/kusama/Cargo.toml b/relay/kusama/Cargo.toml index 0cc6705aab..bb35b40543 100644 --- a/relay/kusama/Cargo.toml +++ b/relay/kusama/Cargo.toml @@ -167,9 +167,9 @@ std = [ "pallet-nomination-pools/std", "pallet-offences-benchmarking?/std", "pallet-offences/std", + "pallet-parameters/std", "pallet-preimage/std", "pallet-proxy/std", - "pallet-parameters/std", "pallet-ranked-collective/std", "pallet-recovery/std", "pallet-referenda/std", @@ -301,6 +301,7 @@ try-runtime = [ "pallet-nis/try-runtime", "pallet-nomination-pools/try-runtime", "pallet-offences/try-runtime", + "pallet-parameters/try-runtime", "pallet-preimage/try-runtime", "pallet-proxy/try-runtime", "pallet-ranked-collective/try-runtime", @@ -320,7 +321,6 @@ try-runtime = [ "polkadot-runtime-common/try-runtime", "runtime-parachains/try-runtime", "sp-runtime/try-runtime", - "pallet-parameters/try-runtime", ] # Enable metadata hash generation at compile time for the `CheckMetadataHash` extension. diff --git a/relay/polkadot/Cargo.toml b/relay/polkadot/Cargo.toml index 6a936c2502..159497e65e 100644 --- a/relay/polkadot/Cargo.toml +++ b/relay/polkadot/Cargo.toml @@ -147,6 +147,7 @@ std = [ "pallet-beefy-mmr/std", "pallet-beefy/std", "pallet-bounties/std", + "pallet-broker/std", "pallet-child-bounties/std", "pallet-conviction-voting/std", "pallet-election-provider-multi-phase/std", @@ -182,6 +183,7 @@ std = [ "pallet-whitelist/std", "pallet-xcm-benchmarks?/std", "pallet-xcm/std", + "polkadot-parachain-primitives/std", "polkadot-primitives/std", "polkadot-runtime-common/std", "polkadot-runtime-constants/std", @@ -212,8 +214,6 @@ std = [ "xcm-executor/std", "xcm-runtime-apis/std", "xcm/std", - "pallet-broker/std", - "polkadot-parachain-primitives/std" ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", @@ -226,6 +226,7 @@ runtime-benchmarks = [ "pallet-bags-list/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-bounties/runtime-benchmarks", + "pallet-broker/runtime-benchmarks", "pallet-child-bounties/runtime-benchmarks", "pallet-conviction-voting/runtime-benchmarks", "pallet-election-provider-multi-phase/runtime-benchmarks", @@ -255,6 +256,7 @@ runtime-benchmarks = [ "pallet-whitelist/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "runtime-parachains/runtime-benchmarks", @@ -263,8 +265,6 @@ runtime-benchmarks = [ "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", "xcm-runtime-apis/runtime-benchmarks", - "pallet-broker/runtime-benchmarks", - "polkadot-parachain-primitives/runtime-benchmarks" ] try-runtime = [ "frame-election-provider-support/try-runtime", diff --git a/system-parachains/coretime/coretime-polkadot/Cargo.toml b/system-parachains/coretime/coretime-polkadot/Cargo.toml index e1a786d326..164d621dfe 100644 --- a/system-parachains/coretime/coretime-polkadot/Cargo.toml +++ b/system-parachains/coretime/coretime-polkadot/Cargo.toml @@ -105,7 +105,6 @@ std = [ "frame-system-rpc-runtime-api/std", "frame-system/std", "frame-try-runtime?/std", - "polkadot-runtime-constants/std", "log/std", "pallet-aura/std", "pallet-authorship/std", @@ -127,6 +126,7 @@ std = [ "polkadot-core-primitives/std", "polkadot-parachain-primitives/std", "polkadot-runtime-common/std", + "polkadot-runtime-constants/std", "scale-info/std", "serde", "serde_json/std", diff --git a/system-parachains/encointer/Cargo.toml b/system-parachains/encointer/Cargo.toml index 0ea8ab3b3d..4deb4218eb 100644 --- a/system-parachains/encointer/Cargo.toml +++ b/system-parachains/encointer/Cargo.toml @@ -141,8 +141,8 @@ runtime-benchmarks = [ "pallet-scheduler/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", - "pallet-xcm/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", + "pallet-xcm/runtime-benchmarks", "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", @@ -201,8 +201,8 @@ std = [ "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transaction-payment/std", "pallet-utility/std", - "pallet-xcm/std", "pallet-xcm-benchmarks?/std", + "pallet-xcm/std", "parachain-info/std", "parachains-common/std", "polkadot-core-primitives/std",