From e0940bd4d32d5876ecee5060eedd11a88f37fd31 Mon Sep 17 00:00:00 2001 From: Cell <1024@lruihao.cn> Date: Mon, 17 Aug 2026 12:47:50 +0800 Subject: [PATCH] fix(layouts): resolve accessibility and SEO issues from AI crawler support (#830) --- hugo.toml | 21 +++++++++++++++++++ layouts/_partials/base/head/index.html | 18 ++++++++++++++++ layouts/baseof.html | 1 - layouts/home.llms.txt | 29 ++++++++++++++++++++++++++ layouts/posts/single.html | 5 +++++ layouts/robots.txt | 18 ++++++++++++++++ 6 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 layouts/home.llms.txt diff --git a/hugo.toml b/hugo.toml index 05f882f9..be41e7cd 100644 --- a/hugo.toml +++ b/hugo.toml @@ -401,6 +401,15 @@ rel = "manifest" isPlainText = true isHTML = false +# Options to make output llms.txt file (AI-friendly site map). +# Add "llms" to home outputs to enable. +[outputFormats.llms] +baseName = "llms" +mediaType = "text/plain" +isPlainText = true +isHTML = false +notAlternative = true + # ------------------------------------------------------------------------------------- # Output Configuration # See: https://gohugo.io/configuration/outputs/ @@ -1157,6 +1166,18 @@ light_theme = "light" dark_theme = "dark" lazy_load = true +# robots.txt configuration (requires enableRobotsTXT = true). +[params.robots] +# Allow AI crawlers (unblock GPTBot/ClaudeBot etc.) + Content-Signal comment + AI hidden hint. +allow_crawlers = false +# Custom rules appended to robots.txt (optional). +# [[params.robots.rules]] +# user_agent = "SomeBot" +# disallow = "/" +# [[params.robots.rules]] +# user_agent = "GoodBot" +# allow = "/" + # Third-party library configuration for some custom CSS and JS files. # Files can be loaded from local assets (relative to /assets/) or remote CDN URLs. # The key is a unique identifier, the value is the file path or URL. diff --git a/layouts/_partials/base/head/index.html b/layouts/_partials/base/head/index.html index fad24ec1..40f51e86 100644 --- a/layouts/_partials/base/head/index.html +++ b/layouts/_partials/base/head/index.html @@ -19,6 +19,24 @@ +{{- /* Robots meta tag: respect page-level robots frontmatter; default to index,follow with max-image-preview:large only when cover exists */ -}} +{{- if isset .Params "robots" -}} + +{{- else -}} + {{- $hasCover := false -}} + {{- if .IsHome -}} + {{- $hasCover = isset .Site.Params.seo "cover" -}} + {{- else if .IsPage -}} + {{- $cover := dict "Page" . | partial "function/get-cover.html" -}} + {{- $hasCover = or $cover.Source $cover.Matches -}} + {{- end -}} + {{- if $hasCover -}} + + {{- else -}} + + {{- end -}} +{{- end -}} + {{- $keywords := .Keywords -}} {{- if not $keywords -}} {{- if .IsPage | and .Params.tags -}} diff --git a/layouts/baseof.html b/layouts/baseof.html index 5ea1ef4a..73232103 100644 --- a/layouts/baseof.html +++ b/layouts/baseof.html @@ -8,7 +8,6 @@ {{- end }} - {{ block "title" . }}{{ .Site.Title }}{{ end }} {{- partial "base/head/index.html" . -}} diff --git a/layouts/home.llms.txt b/layouts/home.llms.txt new file mode 100644 index 00000000..4110b70a --- /dev/null +++ b/layouts/home.llms.txt @@ -0,0 +1,29 @@ +# {{ .Site.Title }} + +> {{ .Site.Params.description }} + +{{- range $name, $taxonomy := .Site.Taxonomies }} +{{- if gt (len $taxonomy) 0 }} + +## {{ title $name }} + +{{- range $term, $pages := $taxonomy }} +{{- if le (len $pages) 5 }} +- {{ $term }}: {{ range $pages }}[{{ .Title }}]({{ .Permalink }}) {{ end }} +{{- else }} +- {{ $term }} ({{ len $pages }}) +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +## Pages + +{{- range where .Site.RegularPages "Draft" "eq" false }} +{{- $page := . }} +{{- with .Description }} +- [{{ $page.Title }}]({{ $page.Permalink }}): {{ . }} +{{- else }} +- [{{ .Title }}]({{ .Permalink }}) +{{- end }} +{{- end }} diff --git a/layouts/posts/single.html b/layouts/posts/single.html index 5186562f..a2d8abf5 100644 --- a/layouts/posts/single.html +++ b/layouts/posts/single.html @@ -182,6 +182,11 @@ {{- /* Content */ -}} {{- $content := dict "Content" .Content "Ruby" (.Param "ruby") "Fraction" (.Param "fraction") "Fontawesome" (.Param "fontawesome") | partial "function/content.html" | safeHTML -}} + {{- if .Site.Params.robots.allow_crawlers -}} + {{- with .OutputFormats.Get "markdown" -}} + + {{- end -}} + {{- end -}}
{{- if not .Params.password -}} {{- $content -}} diff --git a/layouts/robots.txt b/layouts/robots.txt index 3542ef81..660d9ea8 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -9,6 +9,11 @@ Disallow: /*offline/ Disallow: /*404.html$ Disallow: /*.md$ +{{ if .Site.Params.robots.allow_crawlers -}} +# Content-Signal (non-standard, for AI-aware crawlers) +Content-Signal: search=yes, ai-input=yes, ai-train=no +{{- end }} + User-agent: MJ12bot Disallow: / @@ -66,6 +71,7 @@ Disallow: / User-agent: ZoominfoBot Disallow: / +{{ if not .Site.Params.robots.allow_crawlers -}} # OpenAI, ChatGPT User-agent: GPTBot Disallow: / @@ -97,5 +103,17 @@ Disallow: / # Claudebot by Anthropic User-agent: ClaudeBot Disallow: / +{{- end -}} + +{{- range .Site.Params.robots.rules }} + +User-agent: {{ .user_agent }} +{{- with .disallow }} +Disallow: {{ . }} +{{- end }} +{{- with .allow }} +Allow: {{ . }} +{{- end -}} +{{- end -}} Sitemap: {{ "/sitemap.xml" | absURL }}