From 2f5127ae742eb2170e4cb7b2b3a117cf159b8967 Mon Sep 17 00:00:00 2001
From: kez <kez@torproject.org>
Date: Wed, 29 Jun 2022 18:04:42 -0700
Subject: [PATCH] Add an extensible template for robots.txt

---
 content/robots.txt/contents.lr |  5 +++++
 templates/robots.txt           | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 content/robots.txt/contents.lr
 create mode 100644 templates/robots.txt

diff --git a/content/robots.txt/contents.lr b/content/robots.txt/contents.lr
new file mode 100644
index 0000000..0510c37
--- /dev/null
+++ b/content/robots.txt/contents.lr
@@ -0,0 +1,5 @@
+_template: robots.txt
+---
+_discoverable: no
+---
+_model: none
diff --git a/templates/robots.txt b/templates/robots.txt
new file mode 100644
index 0000000..0b087c4
--- /dev/null
+++ b/templates/robots.txt
@@ -0,0 +1,19 @@
+{%- macro capitalize_first(text) -%}
+  {{- text[0].upper() -}}{{- text[1:] -}}
+{%- endmacro -%}
+User-agent: *
+Disallow: /javascript/*
+Disallow: /scss/*
+Disallow: /static/*
+
+{#- the '\n' is because we need a newline,
+but we need to strip trailing and leading whitespace as well.
+fragile, be careful #}
+{% for user_agent, directives in bag('robots_txt').items() -%}
+  {{ '\n' }}User-agent: {{ user_agent }}
+  {%- for directive, values in directives.items() -%}
+    {%- for value in values -%}
+      {{ '\n' }}{{ capitalize_first(directive) }}: {{ value -}}
+    {% endfor -%}
+  {% endfor %}
+{% endfor %}