provide robots.txt via config instead of static file

cubedash/_pages.py

@@ -8,7 +8,7 @@
 from datacube.model import DatasetType, Range
 from datacube.scripts.dataset import build_dataset_info
 from dateutil import tz
-from flask import abort, redirect, request, send_from_directory, url_for
+from flask import abort, redirect, request, url_for
 from werkzeug.datastructures import MultiDict
 from werkzeug.exceptions import HTTPException
 
@@ -51,6 +51,10 @@
 
 _DEFAULT_ARRIVALS_DAYS: int = app.config.get("CUBEDASH_DEFAULT_ARRIVALS_DAY_COUNT", 14)
 
+_ROBOTS_TXT_DEFAULT = (
+    "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**"
+)
+
 # Add server timings to http headers.
 if app.config.get("CUBEDASH_SHOW_PERF_TIMES", False):
     _monitoring.init_app_monitoring()
@@ -676,7 +680,9 @@ def about_page():
 
 @app.route("/robots.txt")
 def robots_txt():
-    return send_from_directory("static", "robots.txt")
+    return utils.as_json(
+        flask.current_app.config.get("ROBOTS_TXT", _ROBOTS_TXT_DEFAULT)
+    )
 
 
 @app.route("/")