# TomHow robots.txt
# https://tomhow.co.uk
# Last updated: 2026-08-23
# Search-tier: OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User,
# Googlebot, Google-Extended, PerplexityBot and Perplexity-User are allowed.
# Training-tier Wave 1: GPTBot, ClaudeBot and CCBot are allowed so future
# models can form a parametric memory of this site. Named groups replace
# the * group, so the wp/admin/staging Disallow rules are repeated.
# Bytespider, Meta, Amazon and Applebot-Extended stay blocked.

# Global default - all unspecified bots may crawl
User-agent: *
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: GPTBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: ChatGPT-User
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: ClaudeBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: Claude-User
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: Claude-SearchBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: anthropic-ai
Disallow: /

User-agent: claude-web
Disallow: /

# Google-Extended is allowed. It governs Gemini and Vertex AI grounding, which
# is a citation pathway, and this site sells work aimed at those answers.
User-agent: Google-Extended
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

# Googlebot is named explicitly so the allow is unambiguous. A named group
# replaces the * group outright for that crawler, it does not inherit from it,
# so the * Disallow rules are repeated here. Without them Googlebot would be
# the one crawler permitted into /admin/, /staging/ and the /wp-* paths.
User-agent: Googlebot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: Perplexity-User
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

User-agent: Applebot-Extended
Disallow: /

# Common Crawl - ALLOW (training-tier discovery; not a citation pathway)
User-agent: CCBot
Allow: /
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /wp-admin/
Disallow: /admin/
Disallow: /staging/

# ByteDance / TikTok - DISALLOW
User-agent: Bytespider
Disallow: /

# Meta - DISALLOW (no B2B citation pathway)
User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Amazon - DISALLOW
User-agent: Amazonbot
Disallow: /

# Sitemap - one canonical URL: the generated index.

Sitemap: https://tomhow.co.uk/sitemap-index.xml