Merge with master

.github/scripts/check-workspace.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+# Ensures that:
+# - all crates are added to the root workspace
+# - local dependencies are resolved via `path`
+#
+# It does not check that the local paths resolve to the correct crate. This is already done by cargo.
+#
+# Must be called with a folder containing a `Cargo.toml` workspace file.
+
+import os
+import sys
+import toml
+import argparse
+
+def parse_args():
+	parser = argparse.ArgumentParser(description='Check Rust workspace integrity.')
+
+	parser.add_argument('workspace_dir', help='The directory to check', metavar='workspace_dir', type=str, nargs=1)
+	parser.add_argument('--exclude', help='Exclude crate paths from the check', metavar='exclude', type=str, nargs='*', default=[])
+
+	args = parser.parse_args()
+	return (args.workspace_dir[0], args.exclude)
+
+def main(root, exclude):
+	workspace_crates = get_members(root, exclude)
+	all_crates = get_crates(root, exclude)
+	print(f'📦 Found {len(all_crates)} crates in total')
+
+	check_duplicates(workspace_crates)
+	check_missing(workspace_crates, all_crates)
+	check_links(all_crates)
+
+# Extract all members from a workspace.
+# Return: list of all workspace paths
+def get_members(workspace_dir, exclude):
+	print(f'🔎 Indexing workspace {os.path.abspath(workspace_dir)}')
+
+	root_manifest_path = os.path.join(workspace_dir, "Cargo.toml")
+	if not os.path.exists(root_manifest_path):
+		print(f'❌ No root manifest found at {root_manifest}')
+		sys.exit(1)
+
+	root_manifest = toml.load(root_manifest_path)
+	if not 'workspace' in root_manifest:
+		print(f'❌ No workspace found in root {root_manifest_path}')
+		sys.exit(1)
+
+	if not 'members' in root_manifest['workspace']:
+		return []
+
+	members = []
+	for member in root_manifest['workspace']['members']:
+		if member in exclude:
+			print(f'❌ Excluded member should not appear in the workspace {member}')
+			sys.exit(1)
+		members.append(member)
+
+	return members
+
+# List all members of the workspace.
+# Return: Map name -> (path, manifest)
+def get_crates(workspace_dir, exclude_crates) -> dict:
+	crates = {}
+
+	for root, dirs, files in os.walk(workspace_dir):
+		if "target" in root:
+			continue
+		for file in files:
+			if file != "Cargo.toml":
+				continue
+
+			path = os.path.join(root, file)
+			with open(path, "r") as f:
+				content = f.read()
+				manifest = toml.loads(content)
+
+			if 'workspace' in manifest:
+				if root != workspace_dir:
+					print("⏩ Excluded recursive workspace at %s" % path)
+				continue
+
+			# Cut off the root path and the trailing /Cargo.toml.
+			path = path[len(workspace_dir)+1:-11]
+			name = manifest['package']['name']
+			if path in exclude_crates:
+				print("⏩ Excluded crate %s at %s" % (name, path))
+				continue
+			crates[name] = (path, manifest)
+
+	return crates
+
+# Check that there are no duplicate entries in the workspace.
+def check_duplicates(workspace_crates):
+	print(f'🔎 Checking for duplicate crates')
+	found = {}
+	for path in workspace_crates:
+		if path in found:
+			print(f'❌ crate is listed twice in the workspace {path}')
+			sys.exit(1)
+		found[path] = True
+
+# Check that all crates are in the workspace.
+def check_missing(workspace_crates, all_crates):
+	print(f'🔎 Checking for missing crates')
+	if len(workspace_crates) == len(all_crates):
+		print(f'✅ All {len(all_crates)} crates are in the workspace')
+		return
+
+	missing = []
+	# Find out which ones are missing.
+	for name, (path, manifest) in all_crates.items():
+		if not path in workspace_crates:
+			missing.append([name, path, manifest])
+	missing.sort()
+
+	for name, path, _manifest in missing:
+		print("❌ %s in %s" % (name, path))
+	print(f'😱 {len(all_crates) - len(workspace_crates)} crates are missing from the workspace')
+	sys.exit(1)
+
+# Check that all local dependencies are good.
+def check_links(all_crates):
+	print(f'🔎 Checking for broken dependency links')
+	links = []
+	broken = []
+
+	for name, (path, manifest) in all_crates.items():
+		def check_deps(deps):
+			for dep in deps:
+				# Could be renamed:
+				dep_name = dep
+				if 'package' in deps[dep]:
+					dep_name = deps[dep]['package']
+				if dep_name in all_crates:
+					links.append((name, dep_name))
+
+					if not 'path' in deps[dep]:
+						broken.append((name, dep_name, "crate must be linked via `path`"))
+						return
+
+		def check_crate(deps):
+			to_checks = ['dependencies', 'dev-dependencies', 'build-dependencies']
+
+			for to_check in to_checks:
+				if to_check in deps:
+					check_deps(deps[to_check])
+
+		# There could possibly target dependant deps:
+		if 'target' in manifest:
+			# Target dependant deps can only have one level of nesting:
+			for _, target in manifest['target'].items():
+				check_crate(target)
+
+		check_crate(manifest)
+
+
+
+	links.sort()
+	broken.sort()
+
+	if len(broken) > 0:
+		for (l, r, reason) in broken:
+			print(f'❌ {l} -> {r} ({reason})')
+
+		print("💥 %d out of %d links are broken" % (len(broken), len(links)))
+		sys.exit(1)
+	else:
+		print("✅ All %d internal dependency links are correct" % len(links))
+
+if __name__ == "__main__":
+	args = parse_args()
+	main(args[0], args[1])

.github/workflows/check-features.yml

@@ -0,0 +1,19 @@
+name: Check Features
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  check-features:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch latest code
+        uses: actions/checkout@v4
+      - name: Check
+        uses: hack-ink/cargo-featalign-action@bea88a864d6ca7d0c53c26f1391ce1d431dc7f34 # v0.1.1
+        with:
+          crate: substrate/bin/node/runtime
+          features: std,runtime-benchmarks,try-runtime
+          ignore: sc-executor
+          default-std: true

.github/workflows/check-workspace.yml

@@ -0,0 +1,21 @@
+name: Check workspace
+
+on:
+  pull_request:
+  merge_group:
+
+jobs:
+  check-workspace:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+
+      - name: install python deps
+        run: pip3 install toml
+
+      - name: check integrity
+        run: >
+            python3 .github/scripts/check-workspace.py .
+            --exclude
+            "substrate/frame/contracts/fixtures/build"
+            "substrate/frame/contracts/fixtures/contracts/common"

.github/workflows/srtool.yml

@@ -0,0 +1,135 @@
+name: Srtool build
+
+env:
+  SUBWASM_VERSION: 0.20.0
+  TOML_CLI_VERSION: 0.2.4
+
+on:
+  push:
+    tags:
+      - "*"
+    branches:
+      - release-v[0-9]+.[0-9]+.[0-9]+*
+      - release-cumulus-v[0-9]+*
+      - release-polkadot-v[0-9]+*
+
+  schedule:
+    - cron: "00 02 * * 1" # 2AM weekly on monday
+
+  workflow_dispatch:
+
+jobs:
+  find-runtimes:
+      name: Scan repo paritytech/polkadot-sdk
+      outputs:
+        runtime: ${{ steps.get_runtimes_list.outputs.runtime }}
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+          with:
+            fetch-depth: 0
+
+        - name: Install tooling
+          run: |
+            URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb
+            curl -L $URL --output toml.deb
+            sudo dpkg -i toml.deb
+            toml --version; jq --version
+
+        - name: Scan runtimes
+          env:
+            EXCLUDED_RUNTIMES: "substrate-test"
+          run: |
+            . ./.github/scripts/common/lib.sh
+
+            echo "Github workspace: ${{ github.workspace }}"
+            echo "Current folder: $(pwd)"; ls -al
+            ls -al
+
+            MATRIX=$(find_runtimes | tee runtimes_list.json)
+            echo $MATRIX
+
+        - name: Get runtimes list
+          id: get_runtimes_list
+          run: |
+            ls -al
+            MATRIX=$(cat runtimes_list.json)
+            echo $MATRIX
+            echo "runtime=$MATRIX" >> $GITHUB_OUTPUT
+
+  srtool:
+    runs-on: ubuntu-latest
+    needs:
+      - find-runtimes
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }}
+
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Srtool build
+        id: srtool_build
+        uses: chevdor/[email protected]
+        with:
+          chain: ${{ matrix.chain }}
+          runtime_dir: ${{ matrix.runtime_dir }}
+
+      - name: Summary
+        run: |
+          echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json
+          cat ${{ matrix.chain }}-srtool-digest.json
+          echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}"
+          echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}"
+
+      # it takes a while to build the runtime, so let's save the artifact as soon as we have it
+      - name: Archive Artifacts for ${{ matrix.chain }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ steps.srtool_build.outputs.wasm }}
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+            ${{ matrix.chain }}-srtool-digest.json
+
+      # We now get extra information thanks to subwasm
+      - name: Install subwasm
+        run: |
+          wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          subwasm --version
+
+      - name: Show Runtime information
+        shell: bash
+        run: |
+          subwasm info ${{ steps.srtool_build.outputs.wasm }}
+          subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }}
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-compressed-info.json
+
+      - name: Extract the metadata
+        shell: bash
+        run: |
+          subwasm meta ${{ steps.srtool_build.outputs.wasm }}
+          subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json
+
+      - name: Check the metadata diff
+        shell: bash
+        # the following subwasm call will error for chains that are not known and/or live, that includes shell for instance
+        run: |
+          subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} || \
+            echo "Subwasm call failed, check the logs. This is likely because ${{ matrix.chain }} is not known by subwasm" | \
+            tee ${{ matrix.chain }}-diff.txt
+
+      - name: Archive Subwasm results
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ matrix.chain }}-info.json
+            ${{ matrix.chain }}-compressed-info.json
+            ${{ matrix.chain }}-metadata.json
+            ${{ matrix.chain }}-diff.txt