Re-work Pex documentation. (pex-tool#2362)

This change moves Pex off RTD and on to GitHub Pages. Instead of
versioned docs out on RTD, only the latest docs will be available via
https://docs.pex-tool.org and versioned docs will be available via
GitHub Releases (for the PDF) and inside the Pex wheel and Pex PEX
themselves via the new `pex3 docs` command that serves the appropriate
docs locally, offline, with full functionality.

With this switch, docs are also now built and link-checked in CI. There
should be no more RTD style misconfiguration production outages.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Noop
         run: "true"
   checks:
-    name:  tox -e format-check,lint-check,typecheck,vendor-check,package
+    name: tox -e format-check,lint-check,typecheck,vendor-check,package,docs
     needs: org-check
     runs-on: ubuntu-22.04
     steps:
@@ -39,10 +39,23 @@ jobs:
         with:
           # We need to keep Python 3.8 for consistent vendoring with tox.
           python-version: "3.8"
-      - name: Check Formatting, Lints, Types, Vendoring and Packaging
+      - name: Check Formatting, Lints and Types
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: format-check,lint-check,typecheck,vendor-check,package -- --additional-format sdist --additional-format wheel
+          tox-env: format-check,lint-check,typecheck
+      - name: Check Vendoring
+        uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
+        with:
+          tox-env: vendor-check
+      - name: Check Packaging
+        uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
+        with:
+          tox-env: package -- --additional-format sdist --additional-format wheel --embed-docs
+      - name: Check Docs
+        uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
+        with:
+          tox-env: docs -- --linkcheck --pdf --clean-html
+
   # N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In
   # particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs.
   linux-tests:

.github/workflows/doc-site.yml

@@ -0,0 +1,42 @@
+name: Deploy Doc Site
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Pex
+        uses: actions/checkout@v3
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Build Doc Site
+        uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
+        with:
+          tox-env: docs -- --linkcheck --pdf --clean-html
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: "dist/docs/html/"
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/release.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Build sdist and wheel
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: package -- --no-pex --additional-format sdist --additional-format wheel
+          tox-env: package -- --no-pex --additional-format sdist --additional-format wheel --embed-docs
       - name: Publish Pex ${{ needs.determine-tag.outputs.release-tag }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
@@ -83,7 +83,11 @@ jobs:
       - name: Package Pex ${{ needs.determine-tag.outputs.release-tag }} PEX
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: package
+          tox-env: package -- --embed-docs
+      - name: Generate Pex ${{ needs.determine-tag.outputs.release-tag }} PDF
+        uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
+        with:
+          tox-env: docs -- --no-html --pdf
       - name: Prepare Changelog
         id: prepare-changelog
         uses: a-scie/actions/[email protected]
@@ -100,6 +104,8 @@ jobs:
           body_path: ${{ steps.prepare-changelog.outputs.changelog-file }}
           draft: false
           prerelease: false
-          files: dist/pex
+          files: |
+            dist/pex
+            dist/docs/pdf/pex.pdf
           fail_on_unmatched_files: true
           discussion_category_name: Announcements

.gitignore

@@ -5,5 +5,5 @@
 /.mypy_cache/
 /.tox/
 /dist/
-/docs/_build
+/docs/_static_dynamic/
 

CHANGES.md

@@ -1,5 +1,18 @@
 # Release Notes
 
+## 2.1.164
+
+This release moves Pex documentation from https://pex.readthedocs.io to
+https://docs.pex-tool.org. While legacy versioned docs will remain
+available at RTD in perpetuity, going forward only the latest Pex
+release docs will be available online at the https://docs.pex-tool.org
+site. If you want to see the Pex docs for the version you are currently
+using, Pex now supports the `pex3 docs` command which will serve the
+docs for your Pex version locally, offline, but with full functionality,
+including search.
+
+* Re-work Pex documentation. (#2362)
+
 ## 2.1.163
 
 This release fixes Pex to work in certain OS / SSL environments where it
@@ -567,7 +580,7 @@ In addition, you can now "inject" runtime environment variables and
 arguments into PEX files such that, when run, the PEX runtime ensures
 those environment variables and command line arguments are passed to the
 PEXed application. See [PEX Recipes](
-https://pex.readthedocs.io/en/latest/recipes.html#uvicorn-and-other-customizable-application-servers
+https://docs.pex-tool.org/recipes.html#uvicorn-and-other-customizable-application-servers
 )
 for more information.
 
@@ -1021,7 +1034,7 @@ PEX venvs (More on additional data files
 as well as a new venv install `--scope` that can be used to create fully
 optimized container images with PEXed applications (See how to use this
 feature
-[here](https://pex.readthedocs.io/en/latest/recipes.html#pex-app-in-a-container)).
+[here](https://docs.pex-tool.org/recipes.html#pex-app-in-a-container)).
 
 * Support splitting venv creation into deps & srcs. (#1634)
 * Fix handling of data files when creating venvs. (#1632)
@@ -1052,7 +1065,7 @@ pre-built wheels are available for that foreign platform.
 
 Additionally, PEXes now know how to set a usable process name when the
 PEX contains the `setproctitle` distribution. See
-[here](https://pex.readthedocs.io/en/v2.1.66/recipes.html#long-running-pex-applications-and-daemons)
+[here](https://docs.pex-tool.org/recipes.html#long-running-pex-applications-and-daemons)
 for more information.
 
 * Add support for `--complete-platform`. (#1609)
@@ -1148,7 +1161,7 @@ creating optimized container images from PEX files.
 ## 2.1.55
 
 This release brings official support for Python 3.10 as well as fixing
-<https://pex.readthedocs.io> doc generation and fixing help for
+<https://docs.pex-tool.org> doc generation and fixing help for
 `pex-tools` / `PEX_TOOLS=1 ./my.pex` pex tools invocations that have too
 few arguments.
 
@@ -1547,7 +1560,7 @@ This release improves interpreter discovery to prefer more recent patch
 versions, e.g. preferring Python 3.6.10 over 3.6.8.
 
 We recently regained access to the docsite, and
-<https://pex.readthedocs.io/en/latest/> is now up-to-date.
+<https://docs.pex-tool.org/> is now up-to-date.
 
 * Prefer more recent patch versions in interpreter discovery. (#1088)
 * Fix `--pex-python` when it's the same as the current interpreter.

CONTRIBUTING.md

@@ -34,7 +34,12 @@ lead to a better end result.
 Before sending off changes you should run `tox -e fmt,lint,check`. This formats, lints and type
 checks the code.
 
-In addition you should run tests, which are divided into integration tests (those under
+If you've made `docs/` changes, you should run `tox -e docs -- --linkcheck --pdf --serve` which
+will build the full doc site including its downloadable PDF version as well as the search index.
+You can browse to the URL printed out at the end of the output to view the site on your local
+machine.
+
+In addition, you should run tests, which are divided into integration tests (those under
 `tests/integration/`) and unit tests (those under `tests/` but not under `tests/integration/`).
 Unit tests have a tox environment name that matches the desired interpreter to test against. So, to
 run unit tests against CPython 3.11 (which you must have installed), use `tox -e py311`. For
@@ -56,3 +61,14 @@ without having to actually install PyPy 2.7 on your machine.
 When you're ready to get additional eyes on your changes, submit a [pull request](
 https://github.com/pex-tool/pex/pulls).
 
+If you've made documentation changes you can render the site in the fork you used for the pull
+request by navigating to the "Deploy Doc Site" action in your fork and running the workflow
+manually. You do this using the "Run workflow" widget in the top row of the workflow run list,
+selecting your PR branch and clicking "Run workflow". This will fail your first time doing this due
+to a branch protection rule the "Deploy Doc Site" action automatically establishes to restrict doc
+site deployments to the main branch. To fix this, navigate to "Environments" in your fork settings
+and edit the "github-pages" branch protection rule, changing "Deployment Branches" from
+"Selected branches" to "All branches" and then save the protection rules. You can now re-run the
+workflow and should be able to browse to https://<your github id>.github.io/pex to browse the
+deployed site with your changes incorporated. N.B.: The site will be destroyed when you delete your
+PR branch.

README.rst

@@ -138,7 +138,7 @@ Documentation
 =============
 
 More documentation about Pex, building .pex files, and how .pex files work
-is available at https://pex.readthedocs.io.
+is available at https://docs.pex-tool.org.
 
 
 Development

build-backend/pex_build/__init__.py

@@ -1,2 +1,11 @@
 # Copyright 2024 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os
+
+# When running under MyPy, this will be set to True for us automatically; so we can use it as a typing module import
+# guard to protect Python 2 imports of typing - which is not normally available in Python 2.
+TYPE_CHECKING = False
+
+
+INCLUDE_DOCS = os.environ.get("__PEX_BUILD_INCLUDE_DOCS__", "False").lower() in ("1", "true")

build-backend/pex_build/hatchling/build.py

@@ -1,7 +1,29 @@
 # Copyright 2024 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
+
+import hatchling.build
+import pex_build
 
 # We re-export all hatchling's PEP-517 build backend hooks here for the build frontend to call.
 from hatchling.build import *  # NOQA
+
+if pex_build.TYPE_CHECKING:
+    from typing import Any, Dict, List, Optional
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    # type: (Optional[Dict[str, Any]]) -> List[str]
+
+    reqs = hatchling.build.get_requires_for_build_wheel(
+        config_settings=config_settings
+    )  # type: List[str]
+    if pex_build.INCLUDE_DOCS:
+        with open("docs-requirements.txt") as fp:
+            for raw_req in fp.readlines():
+                req = raw_req.strip()
+                if not req or req.startswith("#"):
+                    continue
+                reqs.append(req)
+    return reqs

build-backend/pex_build/hatchling/build_hook.py

@@ -0,0 +1,38 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, print_function
+
+import os.path
+import subprocess
+import sys
+
+import pex_build
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+if pex_build.TYPE_CHECKING:
+    from typing import Any, Dict
+
+
+class AdjustBuild(BuildHookInterface):
+    """Allows alteration of the build process."""
+
+    PLUGIN_NAME = "pex-adjust-build"
+
+    def initialize(
+        self,
+        version,  # type: str
+        build_data,  # type: Dict[str, Any]
+    ):
+        # type: (...) -> None
+        if pex_build.INCLUDE_DOCS:
+            out_dir = os.path.join(self.root, "dist", "docs")
+            subprocess.check_call(
+                args=[
+                    sys.executable,
+                    os.path.join(self.root, "scripts", "build_docs.py"),
+                    "--clean-html",
+                    out_dir,
+                ]
+            )
+            build_data["force_include"][out_dir] = os.path.join("pex", "docs")