add robots.txt

cubedash/_pages.py

@@ -8,7 +8,7 @@
 from datacube.model import DatasetType, Range
 from datacube.scripts.dataset import build_dataset_info
 from dateutil import tz
-from flask import abort, redirect, request, url_for
+from flask import abort, redirect, request, send_from_directory, url_for
 from werkzeug.datastructures import MultiDict
 from werkzeug.exceptions import HTTPException
 
@@ -674,6 +674,11 @@ def about_page():
     )
 
 
+@app.route("/robots.txt")
+def robots_txt():
+    return send_from_directory("static", "robots.txt")
+
+
 @app.route("/")
 def default_redirect():
     """Redirect to default starting page."""

integration_tests/test_page_loads.py

@@ -961,6 +961,14 @@ def check_doc_start_has_hint(hint: str, url: str):
     )
 
 
+def test_get_robots(client: FlaskClient):
+    """
+    Check that robots.txt is correctly served from root
+    """
+    text, rv = get_text_response(client, "/robots.txt")
+    assert "User-Agent: *" in text
+
+
 def test_all_give_404s(client: FlaskClient):
     """
     We should get 404 messages, not exceptions, for missing things.