Fix and change a11y tests (#1687)

* Fix and change a11y tests * Apply suggestions from code review Co-authored-by: Tania Allard <[email protected]> * fix lint --------- Co-authored-by: Daniel McCloy <[email protected]> Co-authored-by: Tania Allard <[email protected]>
pydata · Feb 12, 2024 · 355d612 · 355d612
1 parent b119346
commit 355d612
Show file tree

Hide file tree

Showing 33 changed files with 72 additions and 62 deletions.
diff --git a/tests/test_a11y.py b/tests/test_a11y.py
@@ -70,20 +70,69 @@ def url_base():
         process.wait()
 
 
-def fingerprint_violations(accessibility_page_scan_violations):
-    """Create a fingerprint of the Axe violations array.
+def filter_ignored_violations(violations, url_pathname):
+    """Filter out ignored axe-core violations.
 
-    https://playwright.dev/docs/accessibility-testing#using-snapshots-to-allow-specific-known-issues
+    In some tests, we wish to ignore certain accessibility violations that we
+    won't ever fix or that we don't plan to fix soon.
     """
-    return [
-        {
-            "id": violation["id"],
-            "help": violation["help"],
-            "helpUrl": violation["helpUrl"],
-            "targets": [node["target"] for node in violation["nodes"]],
-        }
-        for violation in accessibility_page_scan_violations
-    ]
+    # we allow empty table headers
+    # https://dequeuniversity.com/rules/axe/4.8/empty-table-header?application=RuleDescription
+    if url_pathname == "/examples/pydata.html":
+        return [v for v in violations if v["id"] != "empty-table-header"]
+    elif url_pathname in [
+        "/examples/kitchen-sink/generic.html",
+        "/user_guide/theme-elements.html",
+    ]:
+        filtered = []
+        for violation in violations:
+            # TODO: eventually fix this rule violation. See
+            # https://github.com/pydata/pydata-sphinx-theme/issues/1479.
+            if violation["id"] == "landmark-unique":
+                # Ignore landmark-unique only for .sidebar targets. Don't ignore
+                # it for other targets because then the test might fail to catch
+                # a change that violates the rule in some other way.
+                unexpected_nodes = []
+                for node in violation["nodes"]:
+                    # If some target is not .sidebar then we've found a rule
+                    # violation we weren't expecting
+                    if not all([".sidebar" in target for target in node["target"]]):
+                        unexpected_nodes.append(node)
+                if unexpected_nodes:
+                    violation["nodes"] = unexpected_nodes
+                    filtered.append(violation)
+            else:
+                filtered.append(violation)
+        return filtered
+    else:
+        return violations
+
+
+def format_violations(violations):
+    """Return a pretty string representation of Axe-core violations."""
+    result = f"""
+
+        Found {len(violations)} accessibility violation(s):
+        """
+
+    for violation in violations:
+        result += f"""
+
+            - Rule violated:
+              {violation["id"]} - {violation["help"]}
+                - URL: {violation["helpUrl"]}
+                - Impact: {violation["impact"]}
+                - Tags: {" ".join(violation["tags"])}
+                - Targets:"""
+
+        for node in violation["nodes"]:
+            for target in node["target"]:
+                result += f"""
+                    - {target}"""
+
+        result += "\n\n"
+
+    return result
 
 
 @pytest.mark.a11y
@@ -122,7 +171,7 @@ def fingerprint_violations(accessibility_page_scan_violations):
             "/examples/kitchen-sink/typography.html",
             "#typography",
         ),
-        ("/examples/pydata.html", "#pydata-library-styles"),
+        ("/examples/pydata.html", "#PyData-Library-Styles"),
         (
             "/user_guide/theme-elements.html",
             "#theme-specific-elements",
@@ -142,16 +191,18 @@ def fingerprint_violations(accessibility_page_scan_violations):
         # Using one of the simplest pages on the site, select the whole page for
         # testing in order to effectively test repeated website elements like
         # nav, sidebars, breadcrumbs, footer
-        ("/user_guide/page-toc.html", ""),
+        (
+            "/user_guide/page-toc.html",
+            "",  # select whole page
+        ),
     ],
 )
 def test_axe_core(
-    data_regression,
-    theme: str,
+    page: Page,
     url_base: str,
+    theme: str,
     url_pathname: str,
     selector: str,
-    page: Page,
 ):
     """Should have no Axe-core violations at the provided theme and page section."""
     # Load the page at the provided path
@@ -164,14 +215,13 @@ def test_axe_core(
     # Inject the Axe-core JavaScript library into the page
     page.add_script_tag(path="node_modules/axe-core/axe.min.js")
 
-    # Run the Axe-core library against a section of the page. (Don't run it
-    # against the whole page because in this test we're not trying to find
-    # accessibility violations in the nav, sidebar, footer, or other parts of
-    # the PyData Sphinx Theme documentation website.)
+    # Run the Axe-core library against a section of the page (unless the
+    # selector is empty, then run against the whole page)
     results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')")
 
     # Check found violations against known violations that we do not plan to fix
-    data_regression.check(fingerprint_violations(results["violations"]))
+    filtered_violations = filter_ignored_violations(results["violations"], url_pathname)
+    assert len(filtered_violations) == 0, format_violations(filtered_violations)
 
 
 def test_version_switcher_highlighting(page: Page, url_base: str) -> None:

diff --git a/...11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml b/...11y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_dark_.yml
diff --git a/...1y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml b/...1y/test_axe_core_chromium__examples_kitchen_sink_admonitions_html__admonitions_light_.yml
diff --git a/..._a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml b/..._a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_dark_.yml
diff --git a/...a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml b/...a11y/test_axe_core_chromium__examples_kitchen_sink_api_html__api_documentation_light_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_dark_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_blocks_html__blocks_light_.yml
diff --git a/..._a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml b/..._a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_dark_.yml
diff --git a/...a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml b/...a11y/test_axe_core_chromium__examples_kitchen_sink_generic_html__generic_items_light_.yml
diff --git a/..._a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml b/..._a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_dark_.yml
diff --git a/...a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml b/...a11y/test_axe_core_chromium__examples_kitchen_sink_images_html__images_figures_light_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_dark_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_lists_html__lists_light_.yml
diff --git a/..._axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml b/..._axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_dark_.yml
diff --git a/...axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml b/...axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_2_light_.yml
diff --git a/...st_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml b/...st_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_dark_.yml
diff --git a/...t_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml b/...t_axe_core_chromium__examples_kitchen_sink_structure_html__structural_elements_light_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_dark_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml b/tests/test_a11y/test_axe_core_chromium__examples_kitchen_sink_tables_html__tables_light_.yml
diff --git a/..._a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml b/..._a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_dark_.yml
diff --git a/...a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml b/...a11y/test_axe_core_chromium__examples_kitchen_sink_typography_html__typography_light_.yml
diff --git a/...s/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml b/...s/test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_dark_.yml
diff --git a/.../test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml b/.../test_a11y/test_axe_core_chromium__examples_pydata_html__pydata_library_styles_light_.yml
diff --git a/...est_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml b/...est_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_dark_.yml
diff --git a/...st_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml b/...st_a11y/test_axe_core_chromium__user_guide_extending_html__extending_the_theme_light_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__dark_.yml
diff --git a/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml b/tests/test_a11y/test_axe_core_chromium__user_guide_page_toc_html__light_.yml
diff --git a/...t_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml b/...t_a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_dark_.yml
diff --git a/..._a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml b/..._a11y/test_axe_core_chromium__user_guide_styling_html__theme_variables_and_css_light_.yml
diff --git a/...test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml b/...test_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_dark_.yml
diff --git a/...est_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml b/...est_axe_core_chromium__user_guide_theme_elements_html__theme_specific_elements_light_.yml
diff --git a/...est_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml b/...est_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_dark_.yml
diff --git a/...st_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml b/...st_axe_core_chromium__user_guide_web_components_html__sphinx_design_components_light_.yml