From bb4cab09814215813f7edafb436468ed98711a08 Mon Sep 17 00:00:00 2001
From: "Robin H. Johnson" <robbat2@gentoo.org>
Date: Mon, 10 Jul 2023 22:10:08 -0700
Subject: data/cached: add htaccess file

Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
---
 data/cached/.htaccess | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 data/cached/.htaccess

diff --git a/data/cached/.htaccess b/data/cached/.htaccess
new file mode 100644
index 000000000..89aeaaab1
--- /dev/null
+++ b/data/cached/.htaccess
@@ -0,0 +1,27 @@
+# Allow access to .html/.htmlgz/.json/.yaml/.csv files
+<FilesMatch \.(html|htmlgz|json|yaml|csv)?$>
+  <IfModule mod_version.c>
+    <IfVersion < 2.4>
+      Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+      Require all granted
+    </IfVersion>
+  </IfModule>
+  <IfModule !mod_version.c>
+     Allow from all
+  </IfModule>
+</FilesMatch>
+
+# And no directory listings, either.
+<IfModule mod_version.c>
+  <IfVersion < 2.4>
+    Deny from all
+  </IfVersion>
+  <IfVersion >= 2.4>
+    Require all denied
+  </IfVersion>
+</IfModule>
+<IfModule !mod_version.c>
+  Deny from all
+</IfModule>
-- 
cgit v1.2.3-65-gdbad


From d78c7bcba24958563afaf40096c57c24c80e3100 Mon Sep 17 00:00:00 2001
From: "Robin H. Johnson" <robbat2@gentoo.org>
Date: Mon, 10 Jul 2023 22:12:28 -0700
Subject: data/cached: set headers for bots

Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
---
 data/cached/.htaccess | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/data/cached/.htaccess b/data/cached/.htaccess
index 89aeaaab1..2c6fb2495 100644
--- a/data/cached/.htaccess
+++ b/data/cached/.htaccess
@@ -1,5 +1,12 @@
 # Allow access to .html/.htmlgz/.json/.yaml/.csv files
 <FilesMatch \.(html|htmlgz|json|yaml|csv)?$>
+  # These pages are very large and change a lot, they should be followed, but
+  # not indexed themselves. Ideally we should generate better sitemaps as well,
+  # with only the bugs that are changing.
+  # https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#apache_1
+  Header set X-Robots-Tag "noindex,follow"
+
+  # Set access
   <IfModule mod_version.c>
     <IfVersion < 2.4>
       Allow from all
@@ -11,8 +18,10 @@
   <IfModule !mod_version.c>
      Allow from all
   </IfModule>
+
 </FilesMatch>
 
+# This is for files that don't match.
 # And no directory listings, either.
 <IfModule mod_version.c>
   <IfVersion < 2.4>
-- 
cgit v1.2.3-65-gdbad