[Doc] Limit search engine crawler indexing to latest and master (#31214)

ray-project · Dec 20, 2022 · a15037c · a15037c
1 parent 9b49775
commit a15037c
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -81,6 +81,8 @@
 
 html_extra_path = ["robots.txt"]
 
+html_baseurl = "https://docs.ray.io/en/latest"
+
 # This pattern matches:
 # - Python Repl prompts (">>> ") and it's continuation ("... ")
 # - Bash prompts ("$ ")

diff --git a/doc/source/robots.txt b/doc/source/robots.txt
@@ -1,6 +1,6 @@
 # Algolia-Crawler-Verif: C37A60AFB138256B
 User-agent: *
-Allow: /
+Disallow: /
 Allow: /*/latest/
 Allow: /en/latest/   # Fallback for bots that don't understand wildcards
 Allow: /*/master/