# ----------------------------------------
# GLOBAL RULES
# ----------------------------------------
User-agent: *
Disallow: /act/               # Block memory-heavy routes for unknown bots
Disallow: /*.json             # Block direct JSON access to save RAM
Crawl-delay: 10               # Request a slow crawl (honored by polite bots)

# ----------------------------------------
# 1. SEARCH ENGINES & SEARCH-AI (TRAFFIC GENERATORS)
# These bots help users find you and provide links/citations.
# ----------------------------------------
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Yandex
User-agent: PerplexityBot      # Search AI - Provides links
User-agent: OAI-SearchBot      # SearchGPT - Provides links
User-agent: ChatGPT-User       # Live ChatGPT browsing
Allow: /
Disallow: /act/                # Protect the heavy routes from bulk indexing
Allow: /*.css$                 # Allow CSS for rendering
Allow: /*.js$                  # Allow JS for rendering

# ----------------------------------------
# 2. SOCIAL MEDIA BOTS (LINK PREVIEWS)
# ----------------------------------------
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Applebot
Allow: /
Disallow: /act/

User-agent: meta-externalagent
Disallow: /

# ----------------------------------------
# 3. AI TRAINING & SCRAPERS (BLOCK TO PREVENT CRASHES)
# These consume massive RAM but do not send users to your site.
# ----------------------------------------
User-agent: GPTBot             # OpenAI general crawler
User-agent: Google-Extended    # Google AI training
User-agent: CCBot              # Common Crawl
User-agent: AnthropicAI        # Claude training
User-agent: Claude-Web         # Claude crawler
User-agent: DeepSeekBot        # DeepSeek crawler
User-agent: MetaAI             # Meta AI training
User-agent: ImagesiftBot       # Data scraper
User-agent: Omilex             # Data scraper
Disallow: /                    # Full block to save server resources