{ "@context": "https://agentflare.org/schema", "type": "Article", "tier": "L2-full", "title": "Crawler Policy on the Modern Web: robots.txt and Beyond", "description": "robots.txt remains the standard access signal for crawlers, but it relies on voluntary compliance. Pair per-bot rules that separate training from search…", "canonical": "https://agentflare.org/research/crawler-policy-on-the-modern-web-robotstxt-and-beyond.html", "category": "research", "updated": "2026-10-16", "generated_at": "2026-10-16T00:37:01.652Z", "facts": [ { "label": "Cluster", "value": "Discovery & crawling" }, { "label": "Sources", "value": "8" }, { "label": "Updated", "value": "2026-10-16" } ], "data": { "topic": "robots.txt ai.txt and crawler policy adoption by websites", "cluster": "discovery", "generated_at": "2026-10-16T00:37:01.652Z", "tldr": "robots.txt remains the standard access signal for crawlers, but it relies on voluntary compliance. Pair per-bot rules that separate training from search crawlers with network-level enforcement, and treat llms.txt as a navigation aid rather than an opt-out.", "intro": "", "concepts": [ { "term": "Robots Exclusion Protocol", "definition": "A standard, developed in 1994, by which a robots.txt file at the site root tells crawlers which portions of a website they may visit. It relies on voluntary compliance.", "sources": [ "S5" ] }, { "term": "Training versus search crawlers", "definition": "Major vendors split crawlers by purpose: GPTBot is not OAI-SearchBot, and ClaudeBot is not Claude-SearchBot. Each has its own user-agent and can be controlled independently in robots.txt.", "sources": [ "S8" ] }, { "term": "llms.txt", "definition": "A proposal from Jeremy Howard of Answer.AI (September 2024) framed as helping language models use a site at inference time. It does not control training or crawl permissions.", "sources": [ "S2", "S8" ] }, { "term": "Reverse-proxy crawler blocking", "definition": "Network-level blocking provided by reverse proxies. It is lightly deployed today but offers stronger protection against AI crawlers than robots.txt, with its own limitations.", "sources": [ "S1" ] }, { "term": "Web Bot Auth", "definition": "A signed-request mechanism that determines which automated visitors are treated as legitimate. It is one of three parts of the access layer, alongside llms.txt and infrastructure rules.", "sources": [ "S2" ] } ], "how_it_works": { "title": "Building a layered crawler policy", "steps": [ { "step": "Publish robots.txt at the site root", "detail": "Place the file at the root of the site hierarchy, such as https://www.example.com/robots.txt. Compliant robots fetch it before other files; if it is absent, they assume no limits on crawling." }, { "step": "Write rules per user-agent", "detail": "Address each crawler by its own user-agent string. Because training and search bots are separate, one rule no longer blocks both." }, { "step": "Disallow training, allow search bots", "detail": "A defensible default disallows GPTBot, ClaudeBot, CCBot, Google-Extended and Applebot-Extended, while allowing OAI-SearchBot, Claude-SearchBot and PerplexityBot that send referral traffic." }, { "step": "Enforce with WAF or proxy rules", "detail": "A Cloudflare or firewall rule is enforced before robots.txt is read, so it beats any robots.txt Allow. For non-compliant crawlers such as Bytespider, IP or WAF blocking is the reliable defense." }, { "step": "Verify legitimate bots with signed requests", "detail": "Web Bot Auth determines which automated visitors get treated as legitimate, so infrastructure decisions are not left to whoever last touched the firewall." } ] }, "sections": [ { "h": "What robots.txt does and where it stops", "body": "The Robots Exclusion Protocol was proposed by Martijn Koster in February 1994 and had become a de facto standard by June 1994. It was published as RFC 9309 in September 2022. Compliant robots read the file before fetching anything else, and a missing file is treated as no restriction. The weakness is that compliance is voluntary. Malicious bots can use the file as a directory of pages to visit, and some archival sites ignore it. A study of 203 professional artists found strong demand for tools like robots.txt, but constrained by gaps in technical awareness, agency in deploying them, and limited efficacy against unresponsive crawlers. One practitioner source notes that a robots.txt block can signal where valuable content lives, and recommends pairing it with authentication walls, canonical tagging and noindex headers. Treat it as a request to polite crawlers, not as access control.", "sources": [ "S5", "S1", "S4" ] }, { "h": "Separating training, search and real-time bots", "body": "OpenAI runs GPTBot for training and OAI-SearchBot for ChatGPT search. Anthropic runs three separate bots, and Amazon, Google and Apple each separate training access from search and assistant access. The consequence is that blocking all AI bots as one bucket can remove a site from AI search citations. One source frames the defensible default as disallowing training crawlers while allowing search and retrieval crawlers. Cloudflare data shows why the distinction matters operationally: GPTBot rose from 5% to 30% share of AI crawling between May 2024 and May 2025, and Meta-ExternalAgent entered at 19%. Cloudflare also measured Anthropic crawling roughly 70,900 pages per referred visitor at its June 2025 peak, against about 5:1 for traditional Googlebot. Weigh these crawl-to-referral ratios against your own traffic goals before choosing which bots to allow.", "sources": [ "S8", "S3" ] }, { "h": "Enforcement and verification beyond robots.txt", "body": "Existing tools include robots.txt, NoAI meta tags and active blocking by reverse proxies. Few are designed to resist a resource-rich adversary determined to scrape a site. Researchers who tested network-level blockers from reverse proxies found stronger protection against AI crawlers than robots.txt gives, despite limited deployment, but with their own limitations. Firewall rules sit earlier in the request path: a WAF block is enforced before robots.txt is read. For crawlers that ignore the file, such as Bytespider, IP or WAF blocking is described as the only reliable defense. The access layer also includes Web Bot Auth, a signed-request mechanism that decides which automated visitors are treated as legitimate. One agency source argues that infrastructure rules, not llms.txt, decide whether AI crawlers reach a commerce catalog at all. Review CDN and firewall settings as part of crawler policy.", "sources": [ "S1", "S8", "S2" ] } ], "comparison": { "title": "robots.txt versus llms.txt", "columns": [ "Aspect", "robots.txt", "llms.txt" ], "rows": [ [ "Role", "Governs crawler access to a site", "Markdown guide helping LLMs navigate a site at inference time" ], [ "Training opt-out", "Yes, through per-user-agent rules for bots such as GPTBot", "No; it does not control training or crawl permissions" ], [ "Enforcement", "Voluntary compliance; polite crawlers such as Googlebot and Bingbot follow it", "Self-declared, with nothing verifying the claim" ], [ "Known consumers", "Search and AI crawlers that choose to comply", "Developer-facing systems: coding assistants, agent frameworks, some tracking crawlers" ], [ "Standing", "RFC 9309, published September 2022", "Google says no new machine-readable files are needed for its AI features" ] ], "sources": [ "S8", "S2", "S4", "S5" ] }, "key_numbers": [ { "value": "50–70%", "label": "Share of website traffic attributed to automated crawlers (Akamai, Imperva analyses)", "source": "S1" }, { "value": "5% to 30%", "label": "GPTBot share of AI crawling, May 2024 to May 2025", "source": "S3" }, { "value": "70,900", "label": "Pages crawled per referred visitor by Anthropic at June 2025 peak", "source": "S8" }, { "value": "203", "label": "Professional artists in the crawler-protection user study", "source": "S1" } ], "timeline": [ { "date": "1994-02", "event": "Martijn Koster proposes robots.txt on the www-talk mailing list.", "source": "S5" }, { "date": "2019-07", "event": "Google announces proposal of the Robots Exclusion Protocol as an IETF standard.", "source": "S5" }, { "date": "2022-09", "event": "Robots Exclusion Protocol published as RFC 9309.", "source": "S5" }, { "date": "2024-09", "event": "Jeremy Howard of Answer.AI proposes llms.txt.", "source": "S2" }, { "date": "2026-05", "event": "Chrome Lighthouse adds an agentic browsing audit that checks for llms.txt.", "source": "S2" }, { "date": "2026-06", "event": "Google updates its generative AI search guide, repeating that special files are not needed.", "source": "S2" } ], "pitfalls": [ { "text": "Blocking all AI bots as one bucket can remove a site from AI search citations, because training and search crawlers now use separate user-agents.", "source": "S8" }, { "text": "robots.txt is not privacy or security: it relies on voluntary compliance, and malicious bots can use it as a directory of pages to visit.", "source": "S5" }, { "text": "Do not treat llms.txt as a training opt-out. It guides navigation at inference time and does not control crawl permissions.", "source": "S8" }, { "text": "Do not expect robots.txt to stop non-compliant crawlers such as Bytespider; WAF or IP blocking is the reliable defense for those.", "source": "S8" } ], "takeaways": [ "Use robots.txt as a voluntary signal with separate rules for each training, search and real-time user-agent.", "Back policy with WAF or reverse-proxy rules, since firewall blocks are enforced before robots.txt is read.", "Treat llms.txt as a navigation aid for developer-facing tools, not as an access-control or training opt-out mechanism." ], "faq": [ { "q": "Is robots.txt still relevant for AI crawlers?", "a": "Yes, as one layer. It is standardized as RFC 9309 and polite crawlers follow it, but compliance is voluntary. Some AI crawlers do not respect it, so pair it with firewall or reverse-proxy enforcement." }, { "q": "Does llms.txt stop AI companies from training on my content?", "a": "No. llms.txt is a Markdown guide that helps LLMs navigate a site at inference time. robots.txt governs access; llms.txt governs navigation." }, { "q": "Should I block GPTBot and OAI-SearchBot together?", "a": "They are separate bots with separate user-agents. One source recommends disallowing training crawlers like GPTBot while allowing search crawlers like OAI-SearchBot, so you can keep AI-search citations and referral traffic." }, { "q": "How do I stop crawlers that ignore robots.txt?", "a": "Use network-level controls. A WAF rule is enforced before robots.txt is read, and for non-compliant crawlers such as Bytespider, IP or WAF blocking is the only reliable defense. Reverse proxies offer stronger protection with their own limitations." } ], "sources": [ { "title": "Somesite I Used To Crawl", "url": "https://cseweb.ucsd.edu/~savage/papers/IMC25Crawlers.pdf", "domain": "cseweb.ucsd.edu", "date": "" }, { "title": "llms.txt and Crawler Policy: The AI Search Access Layer | Flatline Agency", "url": "https://www.flatlineagency.com/blog/llms-txt-crawler-policy", "domain": "flatlineagency.com", "date": "" }, { "title": "From Googlebot to GPTBot: Who’s crawling your site in 2025 | Cloudflare Blog", "url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025", "domain": "blog.cloudflare.com", "date": "" }, { "title": "Is robots.txt still relevant? | by jemserish | Medium", "url": "https://medium.com/@jems.erish/is-robots-txt-still-relevant-4b74ac5638b0", "domain": "medium.com", "date": "" }, { "title": "robots.txt - Wikipedia", "url": "https://en.wikipedia.org/wiki/Robots.txt", "domain": "en.wikipedia.org", "date": "" }, { "title": "LLMs.txt & Robots.txt: Optimizing for AI Bots | Goodie", "url": "https://higoodie.com/blog/llms-txt-robots-txt-ai-optimization", "domain": "higoodie.com", "date": "" }, { "title": "AI Crawlers and SEO: Optimization Strategies for Websites | Zeo", "url": "https://zeo.org/resources/blog/ai-crawlers-and-seo-optimization-strategies-for-websites", "domain": "zeo.org", "date": "" }, { "title": "AI Crawler Access Control: The 2026 Decision Matrix", "url": "https://www.digitalapplied.com/blog/ai-crawler-access-control-2026-robots-llms-txt-decision-matrix", "domain": "digitalapplied.com", "date": "" } ] }, "analysis_md": "robots.txt remains the standard access signal for crawlers, but it relies on voluntary compliance. Pair per-bot rules that separate training from search crawlers with network-level enforcement, and treat llms.txt as a navigation aid rather than an opt-out.\n\n\n\n## Key numbers\n\n- **50–70%** — Share of website traffic attributed to automated crawlers (Akamai, Imperva analyses) (https://cseweb.ucsd.edu/~savage/papers/IMC25Crawlers.pdf)\n- **5% to 30%** — GPTBot share of AI crawling, May 2024 to May 2025 (https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025)\n- **70,900** — Pages crawled per referred visitor by Anthropic at June 2025 peak (https://www.digitalapplied.com/blog/ai-crawler-access-control-2026-robots-llms-txt-decision-matrix)\n- **203** — Professional artists in the crawler-protection user study (https://cseweb.ucsd.edu/~savage/papers/IMC25Crawlers.pdf)\n\n## Key concepts\n\n- **Robots Exclusion Protocol:** A standard, developed in 1994, by which a robots.txt file at the site root tells crawlers which portions of a website they may visit. It relies on voluntary compliance.\n- **Training versus search crawlers:** Major vendors split crawlers by purpose: GPTBot is not OAI-SearchBot, and ClaudeBot is not Claude-SearchBot. Each has its own user-agent and can be controlled independently in robots.txt.\n- **llms.txt:** A proposal from Jeremy Howard of Answer.AI (September 2024) framed as helping language models use a site at inference time. It does not control training or crawl permissions.\n- **Reverse-proxy crawler blocking:** Network-level blocking provided by reverse proxies. It is lightly deployed today but offers stronger protection against AI crawlers than robots.txt, with its own limitations.\n- **Web Bot Auth:** A signed-request mechanism that determines which automated visitors are treated as legitimate. It is one of three parts of the access layer, alongside llms.txt and infrastructure rules.\n\n## Building a layered crawler policy\n\n1. **Publish robots.txt at the site root** — Place the file at the root of the site hierarchy, such as https://www.example.com/robots.txt. Compliant robots fetch it before other files; if it is absent, they assume no limits on crawling.\n2. **Write rules per user-agent** — Address each crawler by its own user-agent string. Because training and search bots are separate, one rule no longer blocks both.\n3. **Disallow training, allow search bots** — A defensible default disallows GPTBot, ClaudeBot, CCBot, Google-Extended and Applebot-Extended, while allowing OAI-SearchBot, Claude-SearchBot and PerplexityBot that send referral traffic.\n4. **Enforce with WAF or proxy rules** — A Cloudflare or firewall rule is enforced before robots.txt is read, so it beats any robots.txt Allow. For non-compliant crawlers such as Bytespider, IP or WAF blocking is the reliable defense.\n5. **Verify legitimate bots with signed requests** — Web Bot Auth determines which automated visitors get treated as legitimate, so infrastructure decisions are not left to whoever last touched the firewall.\n\n## What robots.txt does and where it stops\n\nThe Robots Exclusion Protocol was proposed by Martijn Koster in February 1994 and had become a de facto standard by June 1994. It was published as RFC 9309 in September 2022. Compliant robots read the file before fetching anything else, and a missing file is treated as no restriction. The weakness is that compliance is voluntary. Malicious bots can use the file as a directory of pages to visit, and some archival sites ignore it. A study of 203 professional artists found strong demand for tools like robots.txt, but constrained by gaps in technical awareness, agency in deploying them, and limited efficacy against unresponsive crawlers. One practitioner source notes that a robots.txt block can signal where valuable content lives, and recommends pairing it with authentication walls, canonical tagging and noindex headers. Treat it as a request to polite crawlers, not as access control.\n\n## Separating training, search and real-time bots\n\nOpenAI runs GPTBot for training and OAI-SearchBot for ChatGPT search. Anthropic runs three separate bots, and Amazon, Google and Apple each separate training access from search and assistant access. The consequence is that blocking all AI bots as one bucket can remove a site from AI search citations. One source frames the defensible default as disallowing training crawlers while allowing search and retrieval crawlers. Cloudflare data shows why the distinction matters operationally: GPTBot rose from 5% to 30% share of AI crawling between May 2024 and May 2025, and Meta-ExternalAgent entered at 19%. Cloudflare also measured Anthropic crawling roughly 70,900 pages per referred visitor at its June 2025 peak, against about 5:1 for traditional Googlebot. Weigh these crawl-to-referral ratios against your own traffic goals before choosing which bots to allow.\n\n## Enforcement and verification beyond robots.txt\n\nExisting tools include robots.txt, NoAI meta tags and active blocking by reverse proxies. Few are designed to resist a resource-rich adversary determined to scrape a site. Researchers who tested network-level blockers from reverse proxies found stronger protection against AI crawlers than robots.txt gives, despite limited deployment, but with their own limitations. Firewall rules sit earlier in the request path: a WAF block is enforced before robots.txt is read. For crawlers that ignore the file, such as Bytespider, IP or WAF blocking is described as the only reliable defense. The access layer also includes Web Bot Auth, a signed-request mechanism that decides which automated visitors are treated as legitimate. One agency source argues that infrastructure rules, not llms.txt, decide whether AI crawlers reach a commerce catalog at all. Review CDN and firewall settings as part of crawler policy.\n\n## robots.txt versus llms.txt\n\n| Aspect | robots.txt | llms.txt |\n| --- | --- | --- |\n| Role | Governs crawler access to a site | Markdown guide helping LLMs navigate a site at inference time |\n| Training opt-out | Yes, through per-user-agent rules for bots such as GPTBot | No; it does not control training or crawl permissions |\n| Enforcement | Voluntary compliance; polite crawlers such as Googlebot and Bingbot follow it | Self-declared, with nothing verifying the claim |\n| Known consumers | Search and AI crawlers that choose to comply | Developer-facing systems: coding assistants, agent frameworks, some tracking crawlers |\n| Standing | RFC 9309, published September 2022 | Google says no new machine-readable files are needed for its AI features |\n\n## Timeline\n\n- 1994-02: Martijn Koster proposes robots.txt on the www-talk mailing list.\n- 2019-07: Google announces proposal of the Robots Exclusion Protocol as an IETF standard.\n- 2022-09: Robots Exclusion Protocol published as RFC 9309.\n- 2024-09: Jeremy Howard of Answer.AI proposes llms.txt.\n- 2026-05: Chrome Lighthouse adds an agentic browsing audit that checks for llms.txt.\n- 2026-06: Google updates its generative AI search guide, repeating that special files are not needed.\n\n## Common pitfalls\n\n- Blocking all AI bots as one bucket can remove a site from AI search citations, because training and search crawlers now use separate user-agents.\n- robots.txt is not privacy or security: it relies on voluntary compliance, and malicious bots can use it as a directory of pages to visit.\n- Do not treat llms.txt as a training opt-out. It guides navigation at inference time and does not control crawl permissions.\n- Do not expect robots.txt to stop non-compliant crawlers such as Bytespider; WAF or IP blocking is the reliable defense for those.\n\n## Key takeaways\n\n- Use robots.txt as a voluntary signal with separate rules for each training, search and real-time user-agent.\n- Back policy with WAF or reverse-proxy rules, since firewall blocks are enforced before robots.txt is read.\n- Treat llms.txt as a navigation aid for developer-facing tools, not as an access-control or training opt-out mechanism.\n\n## FAQ\n\n**Is robots.txt still relevant for AI crawlers?**\nYes, as one layer. It is standardized as RFC 9309 and polite crawlers follow it, but compliance is voluntary. Some AI crawlers do not respect it, so pair it with firewall or reverse-proxy enforcement.\n\n**Does llms.txt stop AI companies from training on my content?**\nNo. llms.txt is a Markdown guide that helps LLMs navigate a site at inference time. robots.txt governs access; llms.txt governs navigation.\n\n**Should I block GPTBot and OAI-SearchBot together?**\nThey are separate bots with separate user-agents. One source recommends disallowing training crawlers like GPTBot while allowing search crawlers like OAI-SearchBot, so you can keep AI-search citations and referral traffic.\n\n**How do I stop crawlers that ignore robots.txt?**\nUse network-level controls. A WAF rule is enforced before robots.txt is read, and for non-compliant crawlers such as Bytespider, IP or WAF blocking is the only reliable defense. Reverse proxies offer stronger protection with their own limitations.\n", "sources": [ { "title": "Somesite I Used To Crawl", "url": "https://cseweb.ucsd.edu/~savage/papers/IMC25Crawlers.pdf" }, { "title": "llms.txt and Crawler Policy: The AI Search Access Layer | Flatline Agency", "url": "https://www.flatlineagency.com/blog/llms-txt-crawler-policy" }, { "title": "From Googlebot to GPTBot: Who’s crawling your site in 2025 | Cloudflare Blog", "url": "https://blog.cloudflare.com/from-googlebot-to-gptbot-whos-crawling-your-site-in-2025" }, { "title": "Is robots.txt still relevant? | by jemserish | Medium", "url": "https://medium.com/@jems.erish/is-robots-txt-still-relevant-4b74ac5638b0" }, { "title": "robots.txt - Wikipedia", "url": "https://en.wikipedia.org/wiki/Robots.txt" }, { "title": "LLMs.txt & Robots.txt: Optimizing for AI Bots | Goodie", "url": "https://higoodie.com/blog/llms-txt-robots-txt-ai-optimization" }, { "title": "AI Crawlers and SEO: Optimization Strategies for Websites | Zeo", "url": "https://zeo.org/resources/blog/ai-crawlers-and-seo-optimization-strategies-for-websites" }, { "title": "AI Crawler Access Control: The 2026 Decision Matrix", "url": "https://www.digitalapplied.com/blog/ai-crawler-access-control-2026-robots-llms-txt-decision-matrix" } ], "related": [ { "name": "Generative Engine Optimization (GEO): A Primer", "url": "https://agentflare.org/research/generative-engine-optimization-geo-a-primer.html" }, { "name": "llms.txt: The Standard for AI-Readable Sites", "url": "https://agentflare.org/research/llmstxt-the-standard-for-ai-readable-sites.html" }, { "name": "Pay-Per-Crawl and the New Bot Web", "url": "https://agentflare.org/research/pay-per-crawl-and-the-new-bot-web.html" }, { "name": "Structured Data for LLMs: JSON-LD That Agents Actually Use", "url": "https://agentflare.org/research/structured-data-for-llms-json-ld-that-agents-actually-use.html" }, { "name": "HTTP 402 & x402: How AI Agents Pay for Content", "url": "https://agentflare.org/research/http-402-x402-how-ai-agents-pay-for-content.html" }, { "name": "The AI Agent Economy in 2026", "url": "https://agentflare.org/research/the-ai-agent-economy-in-2026.html" } ], "pricing": { "price_usd": 0.02, "method": "402", "endpoint": "https://esa.aisa.one/api/v1/access/verify", "autopay_hint": "set crawlerAutoPrice=true with the X-Crawler-Token header", "onboarding": "https://esa.aisa.one/agent/guide.html" }, "powered_by": "AISA — agent-native search, settlement & delivery (https://aisa.one)" }