Docs: Add robots.txt to only allow indexing of latest and stable (

#6517) Currently, all versions of the documentation are indexed with the result that google searches come up with very outdated versions and the latest version is almost impossible to find. The `robots.txt` now disallows any path from being indexed except for the `latest` and `stable` versions of the documentation. Cherry-pick: 5c1f5d6
aiidateam · Aug 7, 2024 · a492e34 · a492e34
1 parent 740ae20
commit a492e34
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -171,6 +171,9 @@
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
 html_show_copyright = False
 
+# This is to tell search engines to index only stable and latest version
+html_extra_path = ['robots.txt']
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

diff --git a/docs/source/robots.txt b/docs/source/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /projects/aiida-core/en/latest/
+Allow: /projects/aiida-core/en/stable/
+Disallow: /