Merge pull request #2758 from mashehu/switch-to-md-api-docs

Switch to markdown based API and linting error docs

.github/workflows/tools-api-docs.yml

@@ -0,0 +1,41 @@
+name: nf-core/tools dev API docs
+# Run on push and PR to test that docs build
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - nf_core/**/*.py
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      ref_name:
+        description: "The branch or tag to build the API docs for"
+        required: true
+        default: "dev"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  api-docs:
+    name: trigger API docs build on website repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: trigger API docs build
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.nf_core_bot_auth_token }}
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: 'nf-core',
+              repo: 'website',
+              workflow_id: 'add-tools-api-docs.yml',
+              ref: 'main'
+              inputs: {
+                "ref_name": ${{ inputs.ref_name || github.ref_name }}
+              }
+            })

CHANGELOG.md

@@ -14,6 +14,7 @@
 
 - make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733))
 - add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744))
+- Switch to markdown based API and error docs ([#2758](https://github.com/nf-core/tools/pull/2758))
 
 ### Modules
 

docs/api/_src/api/index.md

@@ -1,9 +1,8 @@
 # API Reference
 
 ```{toctree}
-:caption: 'Tests:'
 :glob: true
-:maxdepth: 2
+:maxdepth: 1
 
 *
 ```

docs/api/_src/index.md

@@ -20,5 +20,3 @@ This documentation is for the `nf-core/tools` package.
 - [Module code lint tests](module_lint_tests/index.md) (run by `nf-core modules lint`)
 - [Subworkflow code lint tests](subworkflow_lint_tests/index.md) (run by `nf-core subworkflows lint`)
 - [nf-core/tools Python package API reference](api/index.md)
-- {ref}`genindex`
-- {ref}`modindex`

docs/api/_src/module_lint_tests/environment_yml.md

@@ -0,0 +1,5 @@
+# environment_yml
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.environment_yml
+```

docs/api/_src/module_lint_tests/index.md

@@ -1,9 +1,8 @@
 # Module lint tests
 
 ```{toctree}
-:caption: 'Tests:'
 :glob: true
-:maxdepth: 2
+:maxdepth: 1
 
 *
 ```

docs/api/_src/pipeline_lint_tests/index.md

@@ -1,9 +1,8 @@
 # Pipeline lint tests
 
 ```{toctree}
-:caption: 'Tests:'
 :glob: true
-:maxdepth: 2
+:maxdepth: 1
 
 *
 ```

docs/api/_src/subworkflow_lint_tests/index.md

@@ -1,9 +1,8 @@
 # Subworkflow lint tests
 
 ```{toctree}
-:caption: 'Tests:'
 :glob: true
-:maxdepth: 2
+:maxdepth: 1
 
 *
 ```

docs/api/generate-api-docs.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# allow --force option and also a --release option (which takes a release name, or "all")
+force=false
+releases=()
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -f | --force )
+            force=true
+            ;;
+        -r | --release )
+            shift
+            releases+=("$1")
+            ;;
+        -o | --output )
+            shift
+            output_dir="$1"
+            ;;
+        * )
+            echo "Invalid argument: $1"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+
+# Set the output directory if not set
+if [[ -z "$output_dir" ]]; then
+    output_dir="../src/content/tools/docs"
+fi
+
+# if no release is specified, use all releases
+if [[ ${#releases[@]} -eq 0 ]]; then
+    releases=($(git tag))
+    # add 'dev' to the list of releases
+    releases+=("dev")
+fi
+
+# Loop through each release
+for release in "${releases[@]}"; do
+    # Checkout the release
+    git checkout "$release"
+    echo "_________________________"
+    echo "Generating docs for release: $release"
+    echo "_________________________"
+    git checkout docs/api
+    pip install -r docs/api/requirements.txt --quiet
+    # add the napoleon extension to the sphinx conf.py
+    gsed -i 's/^extensions = \[/extensions = \[\n    "sphinx_markdown_builder",/' docs/api/_src/conf.py
+
+    # run docs/api/make_lint_md.py if it exists
+    # if [[ -f "docs/api/make_lint_md.py" ]]; then
+    #     python docs/api/make_lint_md.py
+    # fi
+
+    find nf_core -name "*.py" | while IFS= read -r file; do
+        # echo "Processing $file"
+
+        # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder
+        gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file"
+
+    done
+
+    # fix syntax in lint/merge_markers.py
+    gsed -i 's/>>>>>>> or <<<<<<</``>>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py
+    # remove markdown files if --force is set
+    if [[ "$force" = true ]]; then
+        echo -e "\n\e[31mRemoving $output_dir/$release because of '--force'\e[0m"
+        rm -rf "$output_dir/$release"
+    fi
+    sphinx-build -b markdown docs/api/_src "$output_dir/$release"
+
+    # undo all changes
+    git restore .
+
+    git checkout -
+    # replace :::{seealso} with :::tip in the markdown files
+    find "$output_dir/$release" -name "*.md" -exec gsed -i 's/:::{seealso}/:::tip/g' {} \;
+    i=1
+    sp="/-\|" # spinner
+    find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do
+        # echo "Processing $file"
+        printf "\b${sp:i++%${#sp}:1}"
+        node docs/api/remark.mjs "$file"
+    done
+    # remove empty files
+    find "$output_dir/$release" -name "*.md" -size 0 -delete
+    # remove `.doctrees` directory
+    rm -rf "$output_dir/$release/.doctrees"
+    # run pre-commit to fix any formatting issues on the generated markdown files
+    pre-commit run --files "$output_dir/$release"
+done

docs/api/remark.mjs

@@ -0,0 +1,93 @@
+import fs from "fs";
+import { remark } from "remark";
+import { visit } from "unist-util-visit";
+
+function remarkDirectives() {
+    return transformer;
+
+    function transformer(tree) {
+        visit(tree, "heading", visitor);
+        visit(tree, "link", visitor);
+    }
+
+    function visitor(node, index, parent) {
+        if (node.depth === 4) {
+            if (["note", "warning"].includes(node.children[0].value?.toLowerCase())) {
+                const type = node.children[0].value.toLowerCase();
+                parent.children.splice(index, 1);
+                parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`;
+                // if second to list parent.children[index].children ends with ":", check if the next node is a code block, if so, add the code block as a child to the current node
+                if (parent.children[index].children.slice(-1)[0]?.value?.trim().endsWith(":")) {
+                    if (parent.children[index + 1].type === "code") {
+                        parent.children[index].children.slice(-1)[0].value += "\n";
+                        parent.children[index].children.push(parent.children[index + 1]);
+                        parent.children.splice(index + 1, 1);
+                    }
+                }
+                parent.children[index].children.push({ type: "text", value: "\n:::" });
+            } else if (node.children[0].type === "emphasis") {
+                node.children[0].children.map((child) => {
+                    if (child.type === "text") {
+                        child.type = "inlineCode";
+                        child.value = child.value?.trim() + "{:python}";
+                    }
+                });
+                // convert the rest of the heading to inline code
+                node.children.slice(1).map((child) => {
+                    if (child.type === "text") {
+                        child.type = "inlineCode";
+                        child.value = child.value?.trim() + "{:python}";
+                    }
+                    if (child.type === "link") {
+                        child.children.map((child) => {
+                            if (child.type === "text") {
+                                child.type = "inlineCode";
+                                child.value = child.value?.trim() + "{:python}";
+                            }
+                        });
+                    }
+                });
+            } else if (node.children[0].type !== "inlineCode") {
+                node.children[0] = {
+                    type: "inlineCode",
+                    value: node.children[0].value?.trim() + "{:python}",
+                };
+            }
+        } else if (node.depth === 3) {
+            node.children.map((child) => {
+                if (child.type === "text") {
+                    child.type = "inlineCode";
+                    child.value = child.value?.trim() + "{:python}";
+                }
+                if (child.type === "link") {
+                    child.children.map((child) => {
+                        if (child.type === "text") {
+                            child.type = "inlineCode";
+                            child.value = child.value?.trim() + "{:python}";
+                        }
+                    });
+                }
+                if (child.type === "emphasis") {
+                    child.children.map((child) => {
+                        if (child.type === "text") {
+                            child.type = "inlineCode";
+                            child.value = child.value?.trim() + "{:python}";
+                        }
+                    });
+                }
+            });
+        }
+        if (node.type === "link") {
+            node.url = node.url.replace(".md", "");
+        }
+    }
+}
+
+let markdown = fs.readFileSync(process.argv[2]);
+
+remark()
+    .use(remarkDirectives)
+    .process(markdown, function (err, file) {
+        if (err) throw err;
+        fs.writeFileSync(process.argv[2], String(file));
+    });