Merge branch 'i7-robots-txt' into 'main'

Add an extensible template for robots.txt

Closes #7

See merge request tpo/web/lego!65

content/robots.txt/contents.lr

@@ -0,0 +1,5 @@
+_template: robots.txt
+---
+_discoverable: no
+---
+_model: none

templates/robots.txt

@@ -0,0 +1,19 @@
+{%- macro capitalize_first(text) -%}
+  {{- text[0].upper() -}}{{- text[1:] -}}
+{%- endmacro -%}
+User-agent: *
+Disallow: /javascript/*
+Disallow: /scss/*
+Disallow: /static/*
+
+{#- the '\n' is because we need a newline,
+but we need to strip trailing and leading whitespace as well.
+fragile, be careful #}
+{% for user_agent, directives in bag('robots_txt').items() -%}
+  {{ '\n' }}User-agent: {{ user_agent }}
+  {%- for directive, values in directives.items() -%}
+    {%- for value in values -%}
+      {{ '\n' }}{{ capitalize_first(directive) }}: {{ value -}}
+    {% endfor -%}
+  {% endfor %}
+{% endfor %}