# robots.txt for azurade.com
#
# Content Signals (https://contentsignals.org/) state how content fetched from
# this site may be used, in addition to whether it may be fetched at all:
#   search   = build a search index and show links and short excerpts
#   ai-input = feed the content into an AI model as input, for example to
#              ground or cite an answer
#   ai-train = train or fine-tune an AI model
# Every group below allows all three. Machine-readable page copies are listed
# in https://azurade.com/llms.txt.

# Everyone else
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/files/
Allow: /api/v2/openapi.json
Allow: /api/v2/models
Disallow: /api/
Disallow: /_/
Disallow: /index.txt
Disallow: /*/index.txt
Allow: /

# AI assistants, AI search and AI training crawlers
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: YouBot
User-agent: Diffbot
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/files/
Allow: /api/v2/openapi.json
Allow: /api/v2/models
Disallow: /api/
Disallow: /_/
Disallow: /index.txt
Disallow: /*/index.txt
Allow: /

# Search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: msnbot
User-agent: AdIdxBot
User-agent: Yandex
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Slurp
User-agent: SeznamBot
User-agent: Qwantbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/files/
Allow: /api/v2/openapi.json
Allow: /api/v2/models
Disallow: /api/
Disallow: /_/
Disallow: /index.txt
Disallow: /*/index.txt
Allow: /

# Link-preview fetchers
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Slackbot-LinkExpanding
User-agent: redditbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /api/files/
Allow: /api/v2/openapi.json
Allow: /api/v2/models
Disallow: /api/
Disallow: /_/
Disallow: /index.txt
Disallow: /*/index.txt
Allow: /

Sitemap: https://azurade.com/sitemap.xml
Sitemap: https://azurade.com/blog/sitemap.xml