fix(layouts): resolve accessibility and SEO issues from AI crawler support (#830)

layouts/_partials/base/head/index.html

@@ -19,6 +19,24 @@
 <meta name="author" content="{{ $author.name }}">
 <meta name="description" content="{{ with .Description }}{{ . }}{{ else }}{{ if .IsPage | and .Summary }}{{ .Summary | plainify }}{{ else }}{{ with .Site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end }}">
 
+{{- /* Robots meta tag: respect page-level robots frontmatter; default to index,follow with max-image-preview:large only when cover exists */ -}}
+{{- if isset .Params "robots" -}}
+  <meta name="robots" content="{{ .Params.robots }}">
+{{- else -}}
+  {{- $hasCover := false -}}
+  {{- if .IsHome -}}
+    {{- $hasCover = isset .Site.Params.seo "cover" -}}
+  {{- else if .IsPage -}}
+    {{- $cover := dict "Page" . | partial "function/get-cover.html" -}}
+    {{- $hasCover = or $cover.Source $cover.Matches -}}
+  {{- end -}}
+  {{- if $hasCover -}}
+    <meta name="robots" content="index, follow, max-image-preview:large">
+  {{- else -}}
+    <meta name="robots" content="index, follow">
+  {{- end -}}
+{{- end -}}
+
 {{- $keywords := .Keywords -}}
 {{- if not $keywords -}}
   {{- if .IsPage | and .Params.tags -}}

layouts/baseof.html

@@ -8,7 +8,6 @@
     {{- end }}
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
-    <meta name="robots" content="noodp" />
     <title>{{ block "title" . }}{{ .Site.Title }}{{ end }}</title>
     {{- partial "base/head/index.html" . -}}
   </head>

layouts/home.llms.txt

@@ -0,0 +1,29 @@
+# {{ .Site.Title }}
+
+> {{ .Site.Params.description }}
+
+{{- range $name, $taxonomy := .Site.Taxonomies }}
+{{- if gt (len $taxonomy) 0 }}
+
+## {{ title $name }}
+
+{{- range $term, $pages := $taxonomy }}
+{{- if le (len $pages) 5 }}
+- {{ $term }}: {{ range $pages }}[{{ .Title }}]({{ .Permalink }}) {{ end }}
+{{- else }}
+- {{ $term }} ({{ len $pages }})
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+## Pages
+
+{{- range where .Site.RegularPages "Draft" "eq" false }}
+{{- $page := . }}
+{{- with .Description }}
+- [{{ $page.Title }}]({{ $page.Permalink }}): {{ . }}
+{{- else }}
+- [{{ .Title }}]({{ .Permalink }})
+{{- end }}
+{{- end }}

layouts/posts/single.html

@@ -182,6 +182,11 @@
 
     {{- /* Content */ -}}
     {{- $content := dict "Content" .Content "Ruby" (.Param "ruby") "Fraction" (.Param "fraction") "Fontawesome" (.Param "fontawesome") | partial "function/content.html" | safeHTML -}}
+    {{- if .Site.Params.robots.allow_crawlers -}}
+      {{- with .OutputFormats.Get "markdown" -}}
+        <div class="sr-only" aria-hidden="true">If you are an AI agent, LLM, or automated tool, the clean Markdown version of this page is available at {{ .Permalink }} — optimized for AI and LLM consumption.</div>
+      {{- end -}}
+    {{- end -}}
     <div class="content" id="content"{{ if $pagefindEnabled }} data-pagefind-body{{ end }}{{ with .Param "end_flag" }} data-end-flag="{{ . }}"{{ end }}>
       {{- if not .Params.password -}}
         {{- $content -}}

layouts/robots.txt

@@ -9,6 +9,11 @@ Disallow: /*offline/
 Disallow: /*404.html$
 Disallow: /*.md$
 
+{{ if .Site.Params.robots.allow_crawlers -}}
+# Content-Signal (non-standard, for AI-aware crawlers)
+Content-Signal: search=yes, ai-input=yes, ai-train=no
+{{- end }}
+
 User-agent: MJ12bot
 Disallow: /
 
@@ -66,6 +71,7 @@ Disallow: /
 User-agent: ZoominfoBot
 Disallow: /
 
+{{ if not .Site.Params.robots.allow_crawlers -}}
 # OpenAI, ChatGPT
 User-agent: GPTBot
 Disallow: /
@@ -97,5 +103,17 @@ Disallow: /
 # Claudebot by Anthropic
 User-agent: ClaudeBot
 Disallow: /
+{{- end -}}
+
+{{- range .Site.Params.robots.rules }}
+
+User-agent: {{ .user_agent }}
+{{- with .disallow }}
+Disallow: {{ . }}
+{{- end }}
+{{- with .allow }}
+Allow: {{ . }}
+{{- end -}}
+{{- end -}}
 
 Sitemap: {{ "/sitemap.xml" | absURL }}