# ===========================================================
# robots.txt — getyouat.com
# Last updated: 2026-04-03
# ===========================================================

# ---------------------------------------------------------------
# Allow all well-behaved crawlers (Google, Bing, etc.)
# ---------------------------------------------------------------
User-agent: *
Allow: /

# Block backend admin and CMS paths from indexing
Disallow: /admin/
Disallow: /admin/blocks/

# Block query-parameter-heavy URLs that create duplicate content
Disallow: /*?*

# Block common non-content system paths
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /.env
Disallow: /config/
Disallow: /tmp/
Disallow: /cache/
Disallow: /logs/

# ---------------------------------------------------------------
# Google-specific rules
# ---------------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /admin/

# Allow Google Image Bot to crawl images for visibility
User-agent: Googlebot-Image
Allow: /images/
Disallow: /admin/

# ---------------------------------------------------------------
# Bing / Microsoft
# ---------------------------------------------------------------
User-agent: Bingbot
Allow: /
Disallow: /admin/

# ---------------------------------------------------------------
# AI crawlers — Allow for GEO (Generative Engine Optimisation)
# GYA actively pursues AI search visibility; keep AI crawlers open
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

# ---------------------------------------------------------------
# Bad / aggressive bots — block
# ---------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ---------------------------------------------------------------
# Sitemap location
# ---------------------------------------------------------------
Sitemap: https://www.getyouat.com/sitemap.xml