User-agent: *

Disallow: /images/
Disallow: /js/
Disallow: /css/

Disallow: /*link/
Disallow: /*offline/
Disallow: /*404.html$
Disallow: /*.md$

{{ if .Site.Params.robots.allow_crawlers -}}
# Content-Signal (non-standard, for AI-aware crawlers)
Content-Signal: search=yes, ai-input=yes, ai-train=no
{{- end }}

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Block SISTRIX
User-agent: SISTRIX Crawler
Disallow: /
User-agent: sistrix
Disallow: /
User-agent: 007ac9
Disallow: /
User-agent: 007ac9 Crawler
Disallow: /

# Block Uptime robot
User-agent: UptimeRobot/2.0
Disallow: /

# Block Ezooms Robot
User-agent: Ezooms Robot
Disallow: /

# Block Perl LWP
User-agent: Perl LWP
Disallow: /

# Block netEstate NE Crawler (+http://www.website-datenbank.de/)
User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
Disallow: /

# Block WiseGuys Robot
User-agent: WiseGuys Robot
Disallow: /

# Block Turnitin Robot
User-agent: Turnitin Robot
Disallow: /

# Block Heritrix
User-agent: Heritrix
Disallow: /

# Block pricepi
User-agent: pimonster
Disallow: /

User-agent: SurdotlyBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

{{ if not .Site.Params.robots.allow_crawlers -}}
# OpenAI, ChatGPT
User-agent: GPTBot
Disallow: /

# Google AI (Bard, etc)
User-agent: Google-Extended
Disallow: /

# Block common crawl
User-agent: CCBot
Disallow: /

# Facebook
User-agent: FacebookBot
Disallow: /

# Cohere.ai
User-agent: cohere-ai
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

# Claudebot by Anthropic
User-agent: ClaudeBot
Disallow: /
{{- end -}}

{{- range .Site.Params.robots.rules }}

User-agent: {{ .user_agent }}
{{- with .disallow }}
Disallow: {{ . }}
{{- end }}
{{- with .allow }}
Allow: {{ . }}
{{- end -}}
{{- end -}}

Sitemap: {{ "/sitemap.xml" | absURL }}
