Package Pex --scies for release.

These are likely better than the Pex PEX for most Pex binary use cases
since they include the management extra and run in `--venv` mode as well
as providing BusyBox support for equal footing of the `pex`, `pex-tools`
and `pex3` top-level commands.

.github/workflows/ci.yml

@@ -53,7 +53,9 @@ jobs:
       - name: Check Packaging
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: package -- --additional-format sdist --additional-format wheel --embed-docs --clean-docs
+          tox-env: >-
+            package -- --additional-format sdist --additional-format wheel --embed-docs --clean-docs
+            --scies --gen-md-table-of-hash-and-size dist/hashes.md
       - name: Check Docs
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:

.github/workflows/release.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Package Pex ${{ needs.determine-tag.outputs.release-tag }} PEX
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: package -- --embed-docs
+          tox-env: package -- --embed-docs --scies --gen-md-table-of-hash-and-size dist/hashes.md
       - name: Generate Pex ${{ needs.determine-tag.outputs.release-tag }} PDF
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
@@ -97,14 +97,20 @@ jobs:
         uses: actions/attest-build-provenance@v1
         with:
           subject-path: |
-            dist/pex
+            dist/pex*
             dist/docs/pdf/pex.pdf
       - name: Prepare Changelog
         id: prepare-changelog
         uses: a-scie/actions/[email protected]
         with:
           changelog-file: ${{ github.workspace }}/CHANGES.md
           version: ${{ needs.determine-tag.outputs.release-version }}
+      - name: Append Hashes to Changelog
+        run: |
+          changelog_tmp="$(mktemp)"
+          cat "${{ steps.prepare-changelog.outputs.changelog-file }}" <(echo '***') dist/hashes.md \
+            > "${changelog_tmp}"
+          mv "${changelog_tmp}" "${{ steps.prepare-changelog.outputs.changelog-file }}"
       - name: Create ${{ needs.determine-tag.outputs.release-tag }} Release
         uses: softprops/action-gh-release@v2
         with:

build-backend/pex_build/hatchling/build_hook.py

@@ -30,7 +30,7 @@ def initialize(
             subprocess.check_call(
                 args=[
                     sys.executable,
-                    os.path.join(self.root, "scripts", "build_docs.py"),
+                    os.path.join(self.root, "scripts", "build-docs.py"),
                     "--clean-html",
                     out_dir,
                 ]

package/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).

package/scie_config.py

@@ -0,0 +1,31 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import pkgutil
+from dataclasses import dataclass
+
+import toml
+
+
+@dataclass(frozen=True)
+class ScieConfig:
+    @classmethod
+    def load(
+        cls, *, pbs_release: str | None = None, python_version: str | None = None
+    ) -> ScieConfig:
+        data = pkgutil.get_data(__name__, "package.toml")
+        assert data is not None, f"Expected to find a sibling package.toml file to {__file__}."
+        scie_config = toml.loads(data.decode())["scie"]
+        return cls(
+            pbs_release=pbs_release or scie_config["pbs-release"],
+            python_version=python_version or scie_config["python-version"],
+            pex_extras=tuple(scie_config["pex-extras"]),
+            platforms=tuple(scie_config["platforms"]),
+        )
+
+    pbs_release: str
+    python_version: str
+    pex_extras: tuple[str, ...]
+    platforms: tuple[str, ...]

scripts/package.py → scripts/create-packages.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+from __future__ import absolute_import, annotations
+
 import atexit
+import glob
 import hashlib
 import io
 import os
@@ -15,7 +18,11 @@
 from pathlib import Path, PurePath
 from typing import Dict, Iterator, Optional, Tuple, cast
 
+from package.scie_config import ScieConfig
+from pex.common import safe_mkdtemp
+
 DIST_DIR = Path("dist")
+PACKAGE_DIR = Path("package")
 
 
 def build_pex_pex(
@@ -53,6 +60,95 @@ def build_pex_pex(
     return output_file
 
 
+def build_pex_scies(
+    scie_dest_dir: Path, verbosity: int = 0, env: Optional[Dict[str, str]] = None
+) -> Iterator[tuple[Path, str]]:
+    scie_config = ScieConfig.load()
+
+    pex_requirement = f".[{','.join(scie_config.pex_extras)}]"
+
+    lock = PACKAGE_DIR / "pex-scie.lock"
+    if not lock.exists():
+        raise SystemExit(
+            f"The Pex scie lock at {lock} does not exist.\n"
+            f"Run `tox -e gen-scie-platform -- --all ...` to generate it."
+        )
+
+    missing_platforms: list[str] = []
+    platforms: list[tuple[str, Path]] = []
+    for platform in scie_config.platforms:
+        complete_platform = PACKAGE_DIR / "complete-platforms" / f"{platform}.json"
+        if not complete_platform.exists():
+            missing_platforms.append(platform)
+        else:
+            platforms.append((platform, complete_platform))
+    if missing_platforms:
+        missing = "\n".join(
+            f"{index}. {missing_platform}"
+            for index, missing_platform in enumerate(missing_platforms, start=1)
+        )
+        raise SystemExit(
+            f"Of the {len(platforms)} expected Pex scie complete platforms, "
+            f"{len(missing)} {'is' if len(missing) == 1 else 'are'} missing:\n{missing}"
+        )
+
+    for platform, complete_platform in platforms:
+        dest_dir = safe_mkdtemp()
+        output_file = os.path.join(dest_dir, "pex")
+        args = [
+            sys.executable,
+            "-m",
+            "pex",
+            *["-v" for _ in range(verbosity)],
+            "--disable-cache",
+            "--no-build",
+            "--no-compile",
+            "--no-use-system-time",
+            "--venv",
+            "--no-strip-pex-env",
+            "--complete-platform",
+            str(complete_platform),
+            "--lock",
+            str(lock),
+            "--scie",
+            "eager",
+            "--scie-only",
+            "--scie-name-style",
+            "platform-file-suffix",
+            "--scie-platform",
+            platform,
+            "--scie-pbs-release",
+            scie_config.pbs_release,
+            "--scie-python-version",
+            scie_config.python_version,
+            "--scie-pbs-stripped",
+            "--scie-hash-alg",
+            "sha256",
+            "--scie-busybox",
+            "@pex",
+            "-o",
+            output_file,
+            "-c",
+            "pex",
+            "--project",
+            pex_requirement,
+        ]
+        subprocess.run(args=args, env=env, check=True)
+
+        artifacts = glob.glob(f"{output_file}*")
+        scie_artifacts = [artifact for artifact in artifacts if not artifact.endswith(".sha256")]
+        if len(scie_artifacts) != 1:
+            raise SystemExit(
+                f"Found unexpected artifacts after generating Pex scie:{os.linesep}"
+                f"{os.linesep.join(sorted(artifacts))}"
+            )
+        scie_name = os.path.basename(scie_artifacts[0])
+        for artifact in artifacts:
+            shutil.move(artifact, scie_dest_dir / os.path.basename(artifact))
+
+        yield scie_dest_dir / scie_name, platform
+
+
 def describe_rev() -> str:
     if not os.path.isdir(".git") and os.path.isfile("PKG-INFO"):
         # We're being build from an unpacked sdist.
@@ -128,22 +224,41 @@ def main(
     embed_docs: bool = False,
     clean_docs: bool = False,
     pex_output_file: Optional[Path] = DIST_DIR / "pex",
+    scie_dest_dir: Optional[Path] = None,
+    markdown_hash_table_file: Optional[Path] = None,
     serve: bool = False
 ) -> None:
     env = os.environ.copy()
     if embed_docs:
         env.update(__PEX_BUILD_INCLUDE_DOCS__="1")
 
+    hash_table: dict[Path, tuple[str, int]] = {}
     if pex_output_file:
         print(f"Building Pex PEX to `{pex_output_file}` ...")
         build_pex_pex(pex_output_file, verbosity, env=env)
 
         rev = describe_rev()
         sha256, size = describe_file(pex_output_file)
+        hash_table[pex_output_file] = sha256, size
         print(f"Built Pex PEX @ {rev}:")
         print(f"sha256: {sha256}")
         print(f"  size: {size}")
 
+    if scie_dest_dir:
+        print(f"Building Pex scies to `{scie_dest_dir}` ...")
+        for scie, platform in build_pex_scies(scie_dest_dir, verbosity, env=env):
+            hash_table[scie] = describe_file(scie)
+            print(f"  Built Pex scie for {platform} at `{scie}`")
+
+    if markdown_hash_table_file and hash_table:
+        with markdown_hash_table_file.open(mode="w") as fp:
+            print("|file|sha256|size|", file=fp)
+            print("|----|------|----|", file=fp)
+            for file, (sha256, size) in sorted(hash_table.items()):
+                print(f"|{file.name}|{sha256}|{size}|", file=fp)
+
+        print(f"Generated markdown table of Pex sizes & hashes at `{markdown_hash_table_file}`")
+
     if additional_dist_formats:
         print(
             f"Building additional distribution formats to `{DIST_DIR}`: "
@@ -214,6 +329,24 @@ def main(
         type=Path,
         help="Build the Pex PEX at this path.",
     )
+    parser.add_argument(
+        "--scies",
+        default=False,
+        action="store_true",
+        help="Build PEX scies.",
+    )
+    parser.add_argument(
+        "--scie-dest-dir",
+        default=DIST_DIR,
+        type=Path,
+        help="Build the Pex scies in this dir.",
+    )
+    parser.add_argument(
+        "--gen-md-table-of-hash-and-size",
+        default=None,
+        type=Path,
+        help="A path to generate a markdown table of packaged asset hashes to.",
+    )
     parser.add_argument(
         "--serve",
         default=False,
@@ -228,5 +361,7 @@ def main(
         embed_docs=args.embed_docs,
         clean_docs=args.clean_docs,
         pex_output_file=None if args.no_pex else args.pex_output_file,
+        scie_dest_dir=args.scie_dest_dir if args.scies else None,
+        markdown_hash_table_file=args.gen_md_table_of_hash_and_size,
         serve=args.serve
     )

scripts/format.py

@@ -41,6 +41,7 @@ def run_black(*args: str) -> None:
                 *args,
                 "build-backend",
                 "docs",
+                "package",
                 "pex",
                 "scripts",
                 "testing",
@@ -54,7 +55,17 @@ def run_black(*args: str) -> None:
 
 def run_isort(*args: str) -> None:
     subprocess.run(
-        args=["isort", *args, "build-backend", "docs", "pex", "scripts", "testing", "tests"],
+        args=[
+            "isort",
+            *args,
+            "build-backend",
+            "docs",
+            "package",
+            "pex",
+            "scripts",
+            "testing",
+            "tests",
+        ],
         check=True,
     )
 

scripts/gen_scie_platform.py → scripts/gen-scie-platform.py

@@ -15,17 +15,17 @@
 import zipfile
 from argparse import ArgumentError, ArgumentTypeError
 from contextlib import contextmanager
-from dataclasses import dataclass
 from pathlib import Path
 from textwrap import dedent
 from typing import IO, Collection, Iterable, Iterator
 
 import github
 import httpx
-import toml
 from github import Github
 from github.WorkflowRun import WorkflowRun
 
+from package.scie_config import ScieConfig
+
 logger = logging.getLogger(__name__)
 
 
@@ -37,25 +37,6 @@ class GitHubError(Exception):
 GEN_SCIE_PLATFORMS_WORKFLOW = "gen-scie-platforms.yml"
 
 
-@dataclass(frozen=True)
-class ScieConfig:
-    @classmethod
-    def load(cls, *, pbs_release: str | None = None, python_version: str | None = None):
-        with (PACKAGE_DIR / "package.toml").open() as fp:
-            scie_config = toml.load(fp)["scie"]
-        return cls(
-            pbs_release=pbs_release or scie_config["pbs-release"],
-            python_version=python_version or scie_config["python-version"],
-            pex_extras=tuple(scie_config["pex-extras"]),
-            platforms=tuple(scie_config["platforms"]),
-        )
-
-    pbs_release: str
-    python_version: str
-    pex_extras: tuple[str, ...]
-    platforms: tuple[str, ...]
-
-
 def create_all_complete_platforms(
     dest_dir: Path,
     scie_config: ScieConfig,

scripts/lint.py

@@ -26,6 +26,7 @@ def run_autoflake(*args: str) -> None:
             "--recursive",
             "build-backend",
             "docs",
+            "package",
             "pex",
             "scripts",
             "testing",

scripts/typecheck.py

@@ -46,7 +46,9 @@ def main() -> None:
         files=sorted(find_files_to_check(include=["docs"])),
         subject="sphinx_pex",
     )
-    run_mypy("3.8", files=sorted(find_files_to_check(include=["scripts"])), subject="scripts")
+    run_mypy(
+        "3.8", files=sorted(find_files_to_check(include=["package", "scripts"])), subject="scripts"
+    )
 
     source_and_tests = sorted(
         find_files_to_check(