# robots.txt for marcusgoll.com

# Default public crawling. Keep operational, auth, and API surfaces out of
# crawler paths while allowing the public aviation site to be indexed.
User-agent: *
Allow: /
Disallow: /admin
Disallow: /manage
Disallow: /api/
Disallow: /dashboard/
Disallow: /maintenance/
Disallow: /marcus/
Disallow: /members/
Disallow: /membership/
Disallow: /sign-in/
Disallow: /sign-up/
Sitemap: https://marcusgoll.com/sitemap.xml

# AI search and user-request retrieval crawlers. These are for citations,
# answer retrieval, and search result quality, not foundation-model training.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /manage
Disallow: /api/
Disallow: /dashboard/
Disallow: /maintenance/
Disallow: /marcus/
Disallow: /members/
Disallow: /membership/
Disallow: /sign-in/
Disallow: /sign-up/

# Legacy/older AI search crawler names kept for compatibility.
User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /manage
Disallow: /api/
Disallow: /dashboard/
Disallow: /maintenance/
Disallow: /marcus/
Disallow: /members/
Disallow: /membership/
Disallow: /sign-in/
Disallow: /sign-up/

# AI training and broad data-collection crawlers.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: FacebookBot
User-agent: Applebot-Extended
User-agent: Amazonbot
Disallow: /

# Non-standard LLM-readable site guide. This is not a Google Search
# requirement; it is a discoverable pointer for agents that look for curated
# context.
LLMS: https://marcusgoll.com/llms.txt