MAINT: use a function to test the warnings of the CI doc build (#1002)

Co-authored-by: Chris Holdgraf <[email protected]>

.github/workflows/prerelease.yml

@@ -36,19 +36,7 @@ jobs:
 
       - name: Check that there are no unexpected Sphinx warnings
         if: matrix.python-version == 3.9
-        shell: python
-        run: |
-          from pathlib import Path
-          import os
-
-          text = Path("./warnings.txt").read_text().strip()
-          print("\n=== Sphinx Warnings ===\n\n" + text)  # Print just for reference so we can look at the logs
-          unexpected = [ii for ii in text.split("\n") if all(i not in ii for i in ["frame.py", "_static", "parse.py"])]
-          print("\n=== Unexpected Warnings ===\n\n" + "\n".join(unexpected))
-
-          env_file = os.getenv('GITHUB_ENV')
-          with open(env_file, "a") as f:
-              f.write("SPHINX_BUILD_UNEXPECTED_WARNINGS = len(unexpected)!=0")
+        run: python tests/check_warnings.py
 
       - name: Run the tests
         run: |

.github/workflows/tests.yml

@@ -62,15 +62,7 @@ jobs:
 
       - name: Check that there are no unexpected Sphinx warnings
         if: matrix.python-version == '3.9'
-        shell: python
-        run: |
-          from pathlib import Path
-          text = Path("./warnings.txt").read_text().strip()
-          print("\n=== Sphinx Warnings ===\n\n" + text)  # Print just for reference so we can look at the logs
-          unexpected = [ii for ii in text.split("\n") if all(i not in ii for i in ["frame.py", "_static", "parse.py"])]
-          for ii in unexpected:
-            print(f"Unexpected warning: \n{unexpected}\n\n")
-          assert len(unexpected) == 0
+        run: python tests/check_warnings.py
 
       - name: Run the tests
         run: pytest --color=yes --cov pydata_sphinx_theme --cov-branch --cov-report term-missing:skip-covered --cov-fail-under ${{ env.COVERAGE_THRESHOLD }}

.gitignore

@@ -64,6 +64,7 @@ instance/
 # Sphinx documentation
 docs/_build/
 docs/examples/generated/
+warnings.txt
 
 # PyBuilder
 target/

docs/community/topics.md

@@ -282,3 +282,19 @@ html_theme_options = {
    "sidebar_secondary": {"remove": "true"}
 }
 ```
+
+## Manage expected build warnings
+
+In our CI workflow, we use a script to check for any warnings raised by Sphinx to assert that the only warnings are _expected_ ones. The list of expected warnings can be found in :code:`tests/warning_list.txt`. To add a new entry, copy/paste the warning message (the message beginning with :code:`WARNING:`) to the bottom of the file.
+
+For example if you get:
+
+```console
+Unexpected warning: C:\hostedtoolcache\windows\Python\3.9.13\x64\lib\site-packages\pandas\core\frame.py:docstring of pandas.core.frame.DataFrame.groupby:42: WARNING: undefined label: 'groupby.transform'
+```
+
+Add the following to the txt file:
+
+```
+WARNING: undefined label: 'groupby.transform'
+```

pyproject.toml

@@ -70,6 +70,7 @@ test = [
 coverage = [
   "pytest-cov",
   "codecov",
+  "colorama",
   "pydata-sphinx-theme[test]",
 ]
 dev = [

tests/check_warnings.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+import sys
+
+from colorama import Fore, init
+
+# init colors for all plateforms
+init()
+
+
+def check_warnings(file):
+    """
+    Check the list of warnings produced by the GitHub CI tests
+    raise errors if there are unexpected ones and/or if some are missing
+
+    Args:
+        file (pathlib.Path): the path to the generated warning.txt file from
+            the CI build
+
+    Return:
+        0 if the warnings are all there
+        1 if some warning are not registered or unexpected
+    """
+
+    # print some log
+    print("\n=== Sphinx Warnings test ===\n")
+
+    # find the file where all the known warnings are stored
+    warning_file = Path(__file__).parent / "warning_list.txt"
+
+    test_warnings = file.read_text().strip().split("\n")
+    ref_warnings = warning_file.read_text().strip().split("\n")
+
+    print(
+        f'Checking build warnings in file: "{file}" and comparing to expected '
+        f'warnings defined in "{warning_file}"\n\n'
+    )
+
+    # find all the missing warnings
+    missing_warnings = []
+    for wa in ref_warnings:
+        index = [i for i, twa in enumerate(test_warnings) if wa in twa]
+        if len(index) == 0:
+            missing_warnings += [wa]
+            print(f"{Fore.YELLOW}Warning was not raised: {Fore.RESET}{wa}\n")
+        else:
+            test_warnings.pop(index[0])
+
+    # the remaining one are unexpected
+    for twa in test_warnings:
+        print(f"{Fore.YELLOW}Unexpected warning: {Fore.RESET}{twa}\n")
+
+    return len(missing_warnings) != 0 or len(test_warnings) != 0
+
+
+if __name__ == "__main__":
+
+    # cast the file to path and resolve to an absolute one
+    file = Path.cwd() / "warnings.txt"
+
+    # execute the test
+    sys.exit(check_warnings(file))

tests/warning_list.txt

@@ -0,0 +1,31 @@
+WARNING: undefined label: 'groupby.transform'
+WARNING: image file not readable: _static/gallery/pandas.png
+WARNING: image file not readable: _static/gallery/numpy.png
+WARNING: image file not readable: _static/gallery/scipy.png
+WARNING: image file not readable: _static/gallery/bokeh.png
+WARNING: image file not readable: _static/gallery/cupy.png
+WARNING: image file not readable: _static/gallery/pyvista.png
+WARNING: image file not readable: _static/gallery/mne-python.png
+WARNING: image file not readable: _static/gallery/networkx.png
+WARNING: image file not readable: _static/gallery/fairlearn.png
+WARNING: image file not readable: _static/gallery/jupyter_book.png
+WARNING: image file not readable: _static/gallery/binder.png
+WARNING: image file not readable: _static/gallery/jupyter.png
+WARNING: image file not readable: _static/gallery/megengine.png
+WARNING: image file not readable: _static/gallery/feature-engine.png
+WARNING: image file not readable: _static/gallery/arviz.png
+WARNING: image file not readable: _static/gallery/sepal.png
+WARNING: image file not readable: _static/gallery/enoslib.png
+WARNING: autosummary: stub file not found 'urllib.parse.DefragResult.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.DefragResult.index'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.DefragResultBytes.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.DefragResultBytes.index'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.ParseResult.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.ParseResult.index'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.ParseResultBytes.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.ParseResultBytes.index'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.SplitResult.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.SplitResult.index'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.SplitResultBytes.count'. Check your autosummary_generate setting.
+WARNING: autosummary: stub file not found 'urllib.parse.SplitResultBytes.index'. Check your autosummary_generate setting.
+ERROR: Unexpected indentation.