# AI Directories Robots.txt
# Last updated: 2026-02-28
#
# IMPORTANT: Googlebot does not support the Crawl-delay directive and will show warnings
# in Google Search Console for any Crawl-delay lines it encounters. These warnings are
# informational only and do not affect crawling. Crawl-delay directives are kept for
# other bots (Bingbot, Slurp, DuckDuckBot, AI crawlers, etc.) that do support them.
# To control Google's crawl rate, use Google Search Console's crawl rate settings.

# Default rule for all bots
User-agent: *
Allow: /
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/*
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
# SEO search pages (e.g. /s/ai+chatbot) - explicitly allowed, no Disallow: /s/
Allow: /s
Allow: /s/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Allow: /popular-searches
Allow: /popular-searches/*

# Disallow access to private/internal pages
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /auth/
Disallow: /test/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/
# Query params allowed so query-driven archives (deals, keyword lists, filter results)
# can be crawled until we migrate to path-based URLs or define canonicals.
# No Disallow for ?page=*, ?search=*, ?q=*, ?query=*

# ============================================================================
# OTHER SEARCH ENGINES (Crawl-delay supported)
# ============================================================================

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 5

# Facebook crawler - block explore to reduce load
User-agent: facebookexternalhit
Crawl-delay: 10

User-agent: meta-externalagent
Crawl-delay: 10

# AI/ML crawlers - allow high-value pages, block infinite/dynamic routes
# This gets us into LLM training data while avoiding ISR cost explosion
# AI crawlers - allow cached/static pages for AI citations, block infinite dynamic routes
User-agent: GPTBot
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: Claude-Web
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

User-agent: Amazonbot
Allow: /$
Allow: /ai-tools
Allow: /ai-tools/
Allow: /ai-tools/categories
Allow: /ai-tools/category/*
Allow: /ai-tools/tags
Allow: /ai-tools/tag/*
Allow: /ai-tools/deals
Allow: /ai-tools/partners
Allow: /ai-tools/affiliates
Allow: /ai-tools/acquire
Allow: /tools
Allow: /tools/*
Allow: /blog
Allow: /blog/*
Allow: /partners
Allow: /partners/*
Allow: /affiliates
Disallow: /api/
Disallow: /dashboard/
Crawl-delay: 5

# Block aggressive/low-value AI crawlers entirely
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Image crawlers
User-agent: Googlebot-Image
Allow: /
Allow: /ai-tools
Allow: /ai-tools/*

User-agent: bingbot-image
Allow: /
Allow: /ai-tools
Allow: /ai-tools/*

# SEO tools - allow for analysis
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

# Note: No global Crawl-delay directive as Googlebot does not support it
# Individual bots that support Crawl-delay have specific directives above

# Sitemaps
Sitemap: https://www.aidirectori.es/sitemap.xml

# LLM crawler guidance (include/exclude, citation)
# https://www.aidirectori.es/llms.txt