Merge branch 'master' into ao-fix-backing-filtering-on-disputes

* master: (106 commits)
  [ci] Delete unused flow (#4676)
  Fix umbrella CI check and fix the C&P message (#4670)
  Add Dockerfiles to the templates (#4637)
  Revamp the Readme of the minimal template (#4649)
  Add chain-spec-builder docker image (#4655)
  Update Amforc bootnodes for Kusama and Polkadot (#4668)
  make all storage items in parachain-system public (#4645)
  [Pools] Refactors and runtime apis for DelegateStake (#4537)
  update amforc westend and its parachain bootnodes (#4641)
  Better error for missing index in CRV2 (#4643)
  Implement `XcmPaymentApi` and `DryRunApi` on all system parachains (#4634)
  Use Unlicense for templates (#4628)
  collator-protocol: remove `elastic-scaling-experimental` feature (#4595)
  Update `runtime_type` ref doc with the new "Associated Type Bounds" (#4624)
  Adds ability to specify chain type in chain-spec-builder (#4542)
  Fix broken windows build (#4636)
  Beefy client generic on aduthority Id (#1816)
  pallet-staking: Put tests behind `cfg(debug_assertions)` (#4620)
  Broker new price adapter (#4521)
  Change `XcmDryRunApi::dry_run_extrinsic` to take a call instead (#4621)
  ...

.config/lychee.toml

@@ -34,11 +34,8 @@ exclude = [
 	"https://github.com/zkcrypto/bls12_381/blob/e224ad4ea1babfc582ccd751c2bf128611d10936/src/test-data/mod.rs",
 	"https://polkadot-try-runtime-node.parity-chains.parity.io/",
 	"https://polkadot.network/the-path-of-a-parachain-block/",
-	"https://research.web3.foundation/en/latest/polkadot/BABE/Babe/#6-practical-results",
 	"https://research.web3.foundation/en/latest/polkadot/NPoS/3.%20Balancing.html",
 	"https://research.web3.foundation/en/latest/polkadot/Token%20Economics.html#inflation-model",
-	"https://research.web3.foundation/en/latest/polkadot/block-production/Babe.html",
-	"https://research.web3.foundation/en/latest/polkadot/block-production/Babe.html#-6.-practical-results",
 	"https://research.web3.foundation/en/latest/polkadot/networking/3-avail-valid.html#topology",
 	"https://research.web3.foundation/en/latest/polkadot/overview/2-token-economics.html",
 	"https://research.web3.foundation/en/latest/polkadot/overview/2-token-economics.html#inflation-model",
@@ -54,5 +51,6 @@ exclude = [
 	"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
 	# 403 rate limited:
 	"https://etherscan.io/block/11090290",
+	"https://subscan.io/",
 	"https://substrate.stackexchange.com/.*",
 ]

.config/zepter.yaml

@@ -25,9 +25,13 @@ workflows:
         '--show-path',
         '--quiet',
       ]
-  # Same as `check`, but with the `--fix` flag.
+  # The umbrella crate uses more features, so we to check those too:
+  check_umbrella:
+    - [ $check.0, '--features=serde,experimental,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
+  # Same as `check_*`, but with the `--fix` flag.
   default:
     - [ $check.0, '--fix' ]
+    - [ $check_umbrella.0, '--fix' ]
 
 # Will be displayed when any workflow fails:
 help:

.forklift/config.toml

@@ -10,6 +10,9 @@ jobsBlackList = []
 logLevel = "warn"
 threadsCount = 6
 
+[cache]
+extraEnv = ["RUNTIME_METADATA_HASH"]
+
 [metrics]
 enabled = true
 pushEndpoint = "placeholder"

.github/scripts/deny-git-deps.py

@@ -0,0 +1,40 @@
+"""
+Script to deny Git dependencies in the Cargo workspace. Can be passed one optional argument for the
+root folder. If not provided, it will use the cwd.
+
+## Usage
+	python3 .github/scripts/deny-git-deps.py polkadot-sdk
+"""
+
+import os
+import sys
+
+from cargo_workspace import Workspace, DependencyLocation
+
+KNOWN_BAD_GIT_DEPS = {
+	'simple-mermaid': ['xcm-docs'],
+	# Fix in <https://github.com/paritytech/polkadot-sdk/issues/2922>
+	'bandersnatch_vrfs': ['sp-core'],
+}
+
+root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
+workspace = Workspace.from_path(root)
+
+def check_dep(dep, used_by):
+	if dep.location != DependencyLocation.GIT:
+		return
+
+	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
+		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
+	else:
+		print(f'🚫 Found git dependency {dep.name} in {used_by}')
+		sys.exit(1)	
+
+# Check the workspace dependencies that can be inherited:
+for dep in workspace.dependencies:
+	check_dep(dep, "workspace")
+
+# And the dependencies of each crate:
+for crate in workspace.crates:
+	for dep in crate.dependencies:
+		check_dep(dep, crate.name)

.github/workflows/check-changed-files.yml

@@ -0,0 +1,57 @@
+# Reusable workflow to perform checks and generate conditions for other workflows. 
+# Currently it checks if any Rust (build-related) file is changed
+# and if the current (caller) workflow file is changed.
+# Example:
+#
+# jobs:
+#   changes:
+#     permissions:
+#       pull-requests: read
+#     uses: ./.github/workflows/check-changed-files.yml
+#   some-job:
+#     needs: changes
+#     if: ${{ needs.changes.outputs.rust }}
+#  ....... 
+
+name: Check changes files
+
+on:
+  workflow_call:
+    # Map the workflow outputs to job outputs
+    outputs:
+      rust: 
+        value: ${{ jobs.changes.outputs.rust }}
+        description: 'true if any of the build-related OR current (caller) workflow files have changed'
+      current-workflow: 
+        value: ${{ jobs.changes.outputs.current-workflow }}
+        description: 'true if current (caller) workflow file has changed'      
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      # true if current workflow (caller) file is changed
+      rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }}
+      current-workflow: ${{ steps.filter.outputs.current-workflow }}
+    steps:
+    - id: current-file
+      run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
+    - run: echo "${{ steps.current-file.outputs.current-workflow-file }}"
+      # For pull requests it's not necessary to checkout the code
+    - id: filter
+      uses: dorny/paths-filter@v3
+      with:
+        predicate-quantifier: 'every'
+        # current-workflow - check if the current (caller) workflow file is changed
+        # rust - check if any Rust (build-related) file is changed
+        filters: |
+          current-workflow:
+            - '${{ steps.current-file.outputs.current-workflow-file }}'
+          rust:
+            - '**/*'
+            - '!.github/**/*'
+            - '!prdoc/**/*'
+            - '!docs/**/*'
+    #

.github/workflows/check-features.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Check
         uses: hack-ink/cargo-featalign-action@bea88a864d6ca7d0c53c26f1391ce1d431dc7f34 # v0.1.1
         with:
-          crate: substrate/bin/node/runtime
+          crate: templates/parachain/runtime/
           features: std,runtime-benchmarks,try-runtime
           ignore: sc-executor
           default-std: true

.github/workflows/check-licenses.yml

@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   check-licenses:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     env:
       LICENSES: "'Apache-2.0' 'GPL-3.0-only' 'GPL-3.0-or-later WITH Classpath-exception-2.0'"

.github/workflows/check-semver.yml

@@ -38,6 +38,7 @@ jobs:
         run: |
           export CARGO_TARGET_DIR=target
           export RUSTFLAGS='-A warnings -A missing_docs'
+          export SKIP_WASM_BUILD=1
           if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc --toolchain nightly-2024-03-01 -v; then
             cat <<EOF
           👋 Hello developer! The SemVer information that you declared in the prdoc file did not match what the CI detected.

.github/workflows/checks-quick.yml

@@ -19,7 +19,7 @@ jobs:
     # GitHub Actions allows using 'env' in a container context.
     # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
     # This workaround sets the container image for each job using 'set-image' job output.
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
@@ -29,7 +29,7 @@ jobs:
       - id: set_image
         run: cat .github/env >> $GITHUB_OUTPUT
   fmt:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     needs: [set-image]
     container:
@@ -39,19 +39,16 @@ jobs:
       - name: Cargo fmt
         run: cargo +nightly fmt --all -- --check
   check-dependency-rules:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
-    # needs: [set-image]
-    # container:
-    #   image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: check dependency rules
         run: |
           cd substrate/
           ../.gitlab/ensure-deps.sh
   check-rust-feature-propagation:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     needs: [set-image]
     container:
@@ -61,7 +58,7 @@ jobs:
       - name: run zepter
         run: zepter run check
   test-rust-features:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     needs: [set-image]
     container:
@@ -71,7 +68,7 @@ jobs:
       - name: run rust features
         run: bash .gitlab/rust-features.sh .
   check-toml-format:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     needs: [set-image]
     container:
@@ -83,22 +80,24 @@ jobs:
           taplo format --check --config .config/taplo.toml
           echo "Please run `taplo format --config .config/taplo.toml` to fix any toml formatting issues"
   check-workspace:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
       - name: install python deps
         run: |
           sudo apt-get update && sudo apt-get install -y python3-pip python3
-          pip3 install toml
+          pip3 install toml "cargo-workspace>=1.2.6"
       - name: check integrity
         run: >
           python3 .github/scripts/check-workspace.py .
           --exclude
           "substrate/frame/contracts/fixtures/build"
           "substrate/frame/contracts/fixtures/contracts/common"
+      - name: deny git deps
+        run: python3 .github/scripts/deny-git-deps.py .
   check-markdown:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - name: Checkout sources
@@ -119,3 +118,32 @@ jobs:
         run: |
           echo "Checking markdown formatting. More info: docs/contributor/markdown_linting.md"
           markdownlint --config "$CONFIG" --ignore target .
+  check-umbrella:
+    runs-on: arc-runners-polkadot-sdk
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+      - name: install python deps
+        run: |
+          sudo apt-get update && sudo apt-get install -y python3-pip python3
+          pip3 install "cargo-workspace>=1.2.4" toml
+      - name: check umbrella correctness
+        run: |
+          python3 scripts/generate-umbrella.py --sdk . --version 0.1.0
+          cargo +nightly fmt --all
+
+          if [ -n "$(git status --porcelain)" ]; then
+            cat <<EOF
+          👋 Hello developer! Apparently you added a new crate that is not part of the umbrella crate?
+
+          You can just apply the patch (git apply PATCH_NAME) that was printed to make this CI check succeed.
+
+          Otherwise feel free to ask in the Merge Request or in Matrix chat.
+          EOF
+
+            git diff
+            exit 1
+          fi

.github/workflows/misc-sync-templates.yml

@@ -104,10 +104,6 @@ jobs:
 
           toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.edition' "$(toml get --raw Cargo.toml 'workspace.package.edition')" > Cargo.temp
           mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
-
-          toml get Cargo.toml 'workspace.lints' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
-
-          toml get Cargo.toml 'workspace.dependencies' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
         working-directory: polkadot-sdk
       - name: Print the result Cargo.tomls for debugging
         if: runner.debug == '1'
@@ -120,6 +116,18 @@ jobs:
       - name: Copy over the new changes
         run: |
           cp -r polkadot-sdk/templates/${{ matrix.template }}/* "${{ env.template-path }}/"
+      - name: Copy over required workspace dependencies
+        run: |
+          echo -e "\n[workspace.dependencies]" >> Cargo.toml
+          set +e
+          # If a workspace dependency is required..
+          while cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`'; do
+            # Get its name..
+            missing_dep=$(cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`' | sed -E 's/(.*)`dependency.(.*)` was not found in `workspace.dependencies`/\2/')
+            # And copy the dependency from the monorepo.
+            toml get ../polkadot-sdk/Cargo.toml 'workspace.dependencies' --output-toml | grep "^${missing_dep} = " >> Cargo.toml
+          done;
+        working-directory: "${{ env.template-path }}"
 
       # 3. Verify the build. Push the changes or create a PR.
 
@@ -148,12 +156,12 @@ jobs:
           token: ${{ steps.app_token.outputs.token }}
           add-paths: |
             ./*
-          title: "[Don't merge] Update the ${{ matrix.template }} template"
+          title: "[Don't merge] Update the ${{ matrix.template }} template to ${{ github.event.inputs.crate_release_version }}"
           body: "The template has NOT been successfully built and needs to be inspected."
-          branch: "update-template/${{ github.event_name }}"
+          branch: "update-template/${{ github.event.inputs.crate_release_version }}"
       - name: Push changes
         run: |
           git add -A .
-          git commit --allow-empty -m "Update template triggered by ${{ github.event_name }}"
+          git commit --allow-empty -m "Update to ${{ github.event.inputs.crate_release_version }} triggered by ${{ github.event_name }}"
           git push
         working-directory: "${{ env.template-path }}"