Fixed sitemap to exclude pages with do-not-crawl tag. Fixed such page…

…s to include correct robots directive.
cagov · Aug 9, 2024 · 37e68c2 · 37e68c2
1 parent ca1f64a
commit 37e68c2
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 11 deletions.
diff --git a/pages/_data/eleventyComputed.js b/pages/_data/eleventyComputed.js
@@ -182,17 +182,15 @@ module.exports = async () => {
   if (!perfAudits) {
     perfAudits = await fetch(perfApiUrl).then((res) => res.json());
   }
-
   return {
     title: (article) => article?.data?.title || article.title,
     id: (article) => article?.data?.id,
     date: (article) => article?.data?.date || article.date,
-    publishdate: (article) =>
-      article?.data?.date.split("T")[0] || article.publishdate,
+    publishdate: (article) => article?.data?.date.split("T")[0] || article.publishdate,
     meta: (article) => article?.data?.excerpt || article.meta,
     description: (article) => article?.data?.excerpt || article.meta,
     author: (article) => article?.data?.author || article.author,
-    tags: (article) => (isPost(article) ? ["news"] : article.tags),
+    tags: (article) => article?.data?.tags || (isPost(article) ? ["news"] : article.tags),
     permalink: (article) => getPermalink(article),
     layout: (article) => getLayout(article),
     eleventyNavigation: (article) => getNavigation(article),

diff --git a/pages/manual-content/sitemap.njk b/pages/manual-content/sitemap.njk
@@ -5,12 +5,12 @@ eleventyExcludeFromCollections: true
 <?xml version="1.0" encoding="utf-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   {% for page in collections.all %}
-    {% if not sitemap.urls[page.url | url ].exclude %}
-      <url>
-        <loc>https://innovation.ca.gov{{ page.url | url }}</loc>
-        <lastmod>{{ page.date.toISOString() }}</lastmod>
-        <changefreq>daily</changefreq>
-      </url>
-    {% endif %}
+      {% if (page.data.tags and not page.data.tags.includes('do-not-crawl')) and not sitemap.urls[page.url | url ].exclude %}
+        <url>
+          <loc>https://innovation.ca.gov{{ page.url | url }}</loc>
+          <lastmod>{{ page.date.toISOString() }}</lastmod>
+          <changefreq>daily</changefreq>
+        </url>
+      {% endif %}
   {% endfor %}
 </urlset>