# ==========================================
# 1. BLOCK AI TRAINING & DATA SCRAPERS
# ==========================================
# These bots scrape content to train LLMs (Large Language Models) 
# without providing SEO/AEO value or direct traffic.

User-agent: meta-externalagent
User-agent: Google-Extended
User-agent: GPTBot
User-agent: CCBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Cohere-ai
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
Disallow: /

# ==========================================
# 2. GLOBAL RULES FOR STANDARD CRAWLERS
# ==========================================
# Allows good search engines (Googlebot, Bingbot, PerplexityBot, OAI-SearchBot) 
# while protecting your WordPress backend and search parameters.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/
# Existing Seaton.giv rules
Disallow: /wp-content/themes/blazetheme/framework/


# ==========================================
# 3. SITEMAP
# ==========================================
Sitemap: https://seaton.gov.uk/sitemap_index.xml