Merge branch 'develop' into update_gdal_version

opendatacube · Jul 23, 2024 · 9ec7530 · 9ec7530
2 parents 943aeea + 09e2e61
commit 9ec7530
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/cubedash/_pages.py b/cubedash/_pages.py
@@ -40,9 +40,12 @@
 
 _DEFAULT_ARRIVALS_DAYS = 14
 
-_ROBOTS_TXT_DEFAULT = (
-    "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**"
-)
+_ROBOTS_TXT_DEFAULT = """User-Agent: *
+Allow: /
+Disallow: /products/*/*
+Disallow: /stac/**
+Disallow: /dataset/*
+"""
 
 
 @bp.route("/<product_name>")