Merge pull request #5501 from rtfd/remove-search-robots-txt

Remove search & API from robots.txt
readthedocs · Mar 20, 2019 · 199480e · 199480e
2 parents a639939 + 7d77435
commit 199480e
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/media/robots.txt b/media/robots.txt
@@ -1,3 +1,9 @@
 User-agent: *
+Disallow: /search/
+Disallow: /api/
 Disallow: /builds/
 Disallow: /sustainability/click/
+
+# This was hitting our site and causing a lot of issues
+User-agent: AhrefsBot 
+Disallow: /