docs: add myst parser

(cherry picked from commit 9eedcac)

docs/Makefile

@@ -25,7 +25,8 @@ setupenv:
 setup:
 	$(POETRY) install
 	$(POETRY) update
-	cp -TLr source $(SOURCEDIR)
+	@if [ ! -d "$(SOURCEDIR)" ]; then mkdir -p "$(SOURCEDIR)"; fi
+	cp -RL source/* $(SOURCEDIR)
 	cd $(SOURCEDIR) && find . -name README.md -execdir mv '{}' index.md ';'
 
 # Clean commands

docs/pyproject.toml

@@ -8,7 +8,6 @@ authors = ["Java Driver Contributors"]
 python = "^3.9"
 pyyaml = "6.0.1"
 pygments = "2.15.1"
-recommonmark = "0.7.1"
 redirects_cli ="~0.1.3"
 sphinx-scylladb-theme = "~1.6.1"
 sphinx-sitemap = "2.5.1"
@@ -17,6 +16,7 @@ Sphinx = "7.2.6"
 sphinx-multiversion-scylla = "~0.3.1"
 setuptools = "^65.6.3"
 wheel = "^0.38.4"
+sphinx-scylladb-markdown = "^0.1.2"
 
 [build-system]
 requires = ["poetry>=0.12"]

docs/source/conf.py

@@ -1,13 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import os
-import sys
-from datetime import date
-import yaml
 import re
-from docutils import nodes
-from recommonmark.transform import AutoStructify
-from recommonmark.parser import CommonMarkParser, splitext, urlparse
+from datetime import date
+from pathlib import Path
 from sphinx_scylladb_theme.utils import multiversion_regex_builder
 from redirects_cli import cli as redirects_cli
 
@@ -47,6 +43,7 @@
     'sphinx.ext.autosectionlabel',
     'sphinx_scylladb_theme',
     'sphinx_multiversion',
+    'sphinx_scylladb_markdown'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -88,6 +85,23 @@
 # Prefix added to all the URLs generated in the 404 page.
 notfound_urls_prefix = ''
 
+# -- Options for markdown extension
+scylladb_markdown_enable = True
+scylladb_markdown_recommonmark_versions = [
+    'scylla-3.7.2.x',
+    'scylla-3.10.2.x',
+    'scylla-3.11.0.x',
+    'scylla-3.11.2.x',
+    'scylla-4.7.2.x',
+    'scylla-4.10.0.x',
+    'scylla-4.11.1.x',
+    'scylla-4.12.0.x',
+    'scylla-4.13.0.x',
+    'scylla-4.14.1.x',
+    'scylla-4.15.0.x',
+]
+suppress_warnings = ["ref.any", "myst.header","myst.xref_missing"]
+
 # -- Options for multiversion extension
 
 # Whitelist pattern for tags
@@ -151,57 +165,20 @@
 
 # Dictionary of values to pass into the template engine’s context for all pages
 html_context = {'html_baseurl': html_baseurl}
-
-# -- Initialize Sphinx
-
-class CustomCommonMarkParser(CommonMarkParser):
-
-    def visit_document(self, node):
-        pass
-
-    def visit_link(self, mdnode):
-        # Override MarkDownParser to avoid checking if relative links exists
-        ref_node = nodes.reference()
-        destination = mdnode.destination
-        _, ext = splitext(destination)
-
-        url_check = urlparse(destination)
-        scheme_known = bool(url_check.scheme)
-
-        if not scheme_known and ext.replace('.', '') in self.supported:
-            destination = destination.replace(ext, '')
-        ref_node['refuri'] = destination
-        ref_node.line = self._get_line(mdnode)
-        if mdnode.title:
-            ref_node['title'] = mdnode.title
-        next_node = ref_node
-
-        self.current_node.append(next_node)
-        self.current_node = ref_node
-
 def replace_relative_links(app, docname, source):
     result = source[0]
     for key in app.config.replacements:
         result = re.sub(key, app.config.replacements[key], result)
     source[0] = result
 
-
-def build_finished(app, exception):
+def redirect_api_page_to_javadoc(app, exception):
     version_name = os.getenv("SPHINX_MULTIVERSION_NAME", "")
     version_name = "/" + version_name if version_name else ""
     redirect_to = version_name +'/api/index.html'
-    out_file = app.outdir +'/api.html'
-    redirects_cli.create(redirect_to=redirect_to,out_file=out_file)
+    out_file = Path(app.outdir) / 'api.html'
+    redirects_cli.create(redirect_to=redirect_to, out_file=str(out_file))
 
 def setup(app):
-    # Setup Markdown parser
-    app.add_source_parser(CustomCommonMarkParser)
-    app.add_config_value('recommonmark_config', {
-        'enable_eval_rst': True,
-        'enable_auto_toc_tree': False,
-    }, True)
-    app.add_transform(AutoStructify)
-
     # Replace DataStax links
     current_slug = os.getenv("SPHINX_MULTIVERSION_NAME", "stable")
     replacements = {
@@ -210,7 +187,4 @@ def setup(app):
     }
     app.add_config_value('replacements', replacements, True)
     app.connect('source-read', replace_relative_links)
-
-    # Create redirect to JavaDoc API
-    app.connect('build-finished', build_finished)
-
+    app.connect('build-finished', redirect_api_page_to_javadoc)

manual/README.md

@@ -34,7 +34,7 @@ Common topics:
 * [Case sensitivity](case_sensitivity/)
 * [OSGi](osgi/)
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/core/README.md

@@ -351,7 +351,7 @@ for (ColumnDefinitions.Definition definition : row.getColumnDefinitions()) {
 
 [CASSANDRA-10145]: https://issues.apache.org/jira/browse/CASSANDRA-10145
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/core/configuration/README.md

@@ -556,7 +556,7 @@ config.getDefaultProfile().getInt(MyCustomOption.AWESOMENESS_FACTOR);
 [HOCON]: https://github.com/typesafehub/config/blob/master/HOCON.md
 [API conventions]: ../../api_conventions
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/core/metadata/README.md

@@ -79,7 +79,7 @@ refreshed. See the [Performance](../performance/#debouncing) page for more detai
 [Metadata]:                                     https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/metadata/Metadata.html
 [Node]:                                         https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/metadata/Node.html
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/core/statements/README.md

@@ -83,7 +83,7 @@ can create execution profiles to capture common combinations of those options).
 [execute]:          https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/session/Session.html#execute-com.datastax.oss.driver.api.core.cql.Statement-
 [executeAsync]:     https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/core/session/Session.html#executeAsync-com.datastax.oss.driver.api.core.cql.Statement-
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/developer/README.md

@@ -37,7 +37,7 @@ from lowest to highest level:
 
 If you're reading this on GitHub, the `.nav` file in each directory contains a suggested order.
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/developer/common/README.md

@@ -27,7 +27,7 @@ This covers utilities or concept that are shared throughout the codebase:
 * the [event bus](event_bus/) is used to decouple some of the internal components through
   asynchronous messaging.
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/mapper/README.md

@@ -172,7 +172,7 @@ You can decide which logs to enable using the standard SLF4J mechanisms (categor
 addition, if you want no logs at all, it's possible to entirely remove them from the generated code
 with the Java compiler option `-Acom.datastax.oss.driver.mapper.logs.enabled=false`.
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/mapper/config/README.md

@@ -133,7 +133,7 @@ You will find the generated files in `build/generated/sources/annotationProcesso
 * [Java 14 records](record/)
 * [Scala](scala/)
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/mapper/daos/README.md

@@ -173,7 +173,7 @@ To control how the hierarchy is scanned, annotate interfaces with [@HierarchySca
 [@HierarchyScanStrategy]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/mapper/annotations/HierarchyScanStrategy.html
 [Entity Inheritance]: ../entities/#inheritance
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/query_builder/README.md

@@ -213,7 +213,7 @@ For a complete tour of the API, browse the child pages in this manual:
 [DseQueryBuilder]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/dse/driver/api/querybuilder/DseQueryBuilder.html
 [DseSchemaBuilder]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/dse/driver/api/querybuilder/DseSchemaBuilder.html
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob:

manual/query_builder/schema/README.md

@@ -65,7 +65,7 @@ element type:
 
 [SchemaBuilder]: https://docs.datastax.com/en/drivers/java/4.17/com/datastax/oss/driver/api/querybuilder/SchemaBuilder.html
 
-```eval_rst
+```{eval-rst}
 .. toctree::
    :hidden:
    :glob: