# AgenticWebScore — robots.txt
# Policy rationale: https://agenticwebscore.com/ai-policy
# Named AI agents are explicitly welcome. There is no wildcard block.
# /api/ is disallowed in every group: a GET there can start a scan of another site.

User-agent: *
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

# Named AI agents
User-agent: GPTBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: OAI-SearchBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: ChatGPT-User
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: ClaudeBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Claude-User
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: anthropic-ai
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: PerplexityBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Perplexity-User
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Google-Extended
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Applebot-Extended
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: CCBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Bytespider
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Amazonbot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Meta-ExternalAgent
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: cohere-ai
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: DuckAssistBot
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: MistralAI-User
Disallow: /api/
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

# Our own scanner identifies itself as:
#   AgenticWebScore/0.3 (+https://agenticwebscore.com/bot)
# It does not consult any target's robots.txt, this file included. That is
# deliberate and it is stated on /abuse: a rubric that measured whether a site
# permits measurement, by asking the site for permission, could only ever measure
# sites that permit it. This file governs agents that visit THIS site; the lines
# above are what we ask of them, not what our scanner follows. To refuse the
# scanner, block the user agent named here. A site that blocks it is recorded as
# not measured, never as a low score.

Sitemap: https://agenticwebscore.com/sitemap.xml