{ "updated": "2026-09-04", "note": "Canonical AnswerScope AI-crawler registry. Every fact is checked against the operator's primary documentation and carries a confidence label and a last-verified date. 'documented' = stated in the vendor's own docs; 'observed' = reported behavior not fully documented. This file is the single source both the /ai-crawlers pages and /data/ai-crawlers.json render from.", "categories": { "discovery": "Discovery / citation crawler, decides whether you can appear and be cited in a live AI answer or AI search result. Allow these to be eligible.", "user": "User-initiated fetcher, visits a page only when a person asks the assistant about it. Often ignores robots.txt because it acts on a direct user request.", "training": "Training crawler, gathers content to train or ground a model. Blocking one opts you out of training; it does NOT remove you from live AI answers." }, "crawlers": [ { "slug": "oai-searchbot", "token": "OAI-SearchBot", "company": "OpenAI", "product": "ChatGPT search", "category": "discovery", "robotsToken": "OAI-SearchBot", "affectsAiAnswers": "yes", "affectsSearch": null, "affectsTraining": false, "verdict": "allow-to-be-cited", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://developers.openai.com/api/docs/bots", "docLabel": "OpenAI, Overview of OpenAI Crawlers", "summary": "OpenAI's crawler for surfacing websites in ChatGPT's search results. Allow it to be eligible for citations and links in ChatGPT search.", "detail": [ "OAI-SearchBot is the crawler OpenAI uses to surface websites in the search features of ChatGPT. OpenAI's documentation states it is \"used to surface websites in search results in ChatGPT's search features.\"", "If you block OAI-SearchBot in robots.txt, your pages are not eligible to appear or be cited in ChatGPT search answers. This is the crawler to allow if being cited by ChatGPT matters to you.", "OAI-SearchBot is distinct from GPTBot, which is for model training, and from ChatGPT-User, which fetches a page only when a person asks about it. Allowing OAI-SearchBot does not, on its own, opt you into model training." ] }, { "slug": "gptbot", "token": "GPTBot", "company": "OpenAI", "product": "OpenAI model training", "category": "training", "robotsToken": "GPTBot", "affectsAiAnswers": "no", "affectsSearch": null, "affectsTraining": true, "verdict": "your-choice-training", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://developers.openai.com/api/docs/bots", "docLabel": "OpenAI, Overview of OpenAI Crawlers", "summary": "OpenAI's model-training crawler. Blocking it opts your content out of training, it does NOT remove you from ChatGPT search answers.", "detail": [ "GPTBot is the crawler OpenAI uses to collect content that may improve its generative AI foundation models. OpenAI states GPTBot is \"used to make our generative AI foundation models more useful and safe,\" and that \"disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models.\"", "GPTBot is a training control, not a discovery crawler. Blocking GPTBot does not remove you from ChatGPT's live search answers, that is governed by OAI-SearchBot. Whether to allow GPTBot is a separate decision about model training.", "A common mistake is treating GPTBot as \"the ChatGPT crawler.\" It is not the crawler that decides citation eligibility in ChatGPT search." ] }, { "slug": "chatgpt-user", "token": "ChatGPT-User", "company": "OpenAI", "product": "ChatGPT (user-initiated)", "category": "user", "robotsToken": "ChatGPT-User", "affectsAiAnswers": "indirect", "affectsSearch": null, "affectsTraining": false, "verdict": "user-initiated", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://developers.openai.com/api/docs/bots", "docLabel": "OpenAI, Overview of OpenAI Crawlers", "summary": "OpenAI's user-initiated fetcher. It visits a page only when a person asks ChatGPT about it, not an automatic crawler.", "detail": [ "ChatGPT-User is the agent OpenAI uses when a person asks ChatGPT (or a custom GPT) a question that requires visiting a specific web page. OpenAI states that \"when users ask ChatGPT or a CustomGPT a question, it may visit a web page with a ChatGPT-User agent.\"", "It is not an automatic crawler and does not gather training data or decide search-index eligibility. It fetches on demand, on behalf of a single user.", "Allowing ChatGPT-User lets ChatGPT read a page a user explicitly points it at; blocking it prevents that on-demand fetch." ] }, { "slug": "googlebot", "token": "Googlebot", "company": "Google", "product": "Google Search + AI Overviews", "category": "discovery", "robotsToken": "Googlebot", "affectsAiAnswers": "yes", "affectsSearch": true, "affectsTraining": false, "verdict": "allow-to-be-cited", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "docLabel": "Google, Common crawlers", "summary": "Google's search crawler. It powers Google Search, and AI Overviews / AI Mode draw from the same Search index, so Googlebot access gates both.", "detail": [ "Googlebot is the crawler for Google Search, it crawls and indexes pages for ranking. Google's AI Overviews and AI Mode are part of Search and draw on the Google Search index rather than a separate crawler, so Googlebot access is what makes a page eligible to be surfaced there.", "Blocking Googlebot removes you from Google Search and, by extension, from AI Overviews. There is no separate crawler you allow to appear in AI Overviews.", "Do not confuse Googlebot with Google-Extended. Google-Extended is a training/grounding control for Gemini and, per Google, does not affect Google Search inclusion at all." ] }, { "slug": "google-extended", "token": "Google-Extended", "company": "Google", "product": "Gemini training + grounding", "category": "training", "robotsToken": "Google-Extended", "affectsAiAnswers": "no", "affectsSearch": false, "affectsTraining": true, "verdict": "your-choice-training", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "docLabel": "Google, Common crawlers", "summary": "A Google control token for Gemini training and grounding. Google states it does NOT affect inclusion or ranking in Google Search.", "detail": [ "Google-Extended is a robots.txt token publishers use to control whether their content is used to train future Gemini models and for grounding in Gemini apps. It is a control token, not a distinct crawling agent, Googlebot does the crawling.", "Google's documentation is explicit: \"Google-Extended does not impact a site's inclusion in Google Search nor is it used as a ranking signal in Google Search.\" Blocking it does not remove you from Google Search or AI Overviews.", "Treating Google-Extended as the crawler that \"feeds Google's AI answers\" is a common and costly error. It governs training and grounding, not Search visibility." ] }, { "slug": "perplexitybot", "token": "PerplexityBot", "company": "Perplexity", "product": "Perplexity search", "category": "discovery", "robotsToken": "PerplexityBot", "affectsAiAnswers": "yes", "affectsSearch": null, "affectsTraining": false, "verdict": "allow-to-be-cited", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://docs.perplexity.ai/guides/bots", "docLabel": "Perplexity, Crawlers & user agents", "summary": "Perplexity's search crawler. Allow it to have the best chance of appearing and being cited in Perplexity answers.", "detail": [ "PerplexityBot is designed to surface and link websites in search results on Perplexity. Perplexity's documentation notes that sites which allow this crawler have the best chance of appearing in Perplexity results.", "PerplexityBot does not collect training data for foundation models, Perplexity states its crawlers serve search and user-facing features only.", "It is separate from Perplexity-User, the on-demand fetcher used when a person asks Perplexity a question directly." ] }, { "slug": "perplexity-user", "token": "Perplexity-User", "company": "Perplexity", "product": "Perplexity (user-initiated)", "category": "user", "robotsToken": "Perplexity-User", "affectsAiAnswers": "indirect", "affectsSearch": null, "affectsTraining": false, "verdict": "user-initiated", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://docs.perplexity.ai/guides/bots", "docLabel": "Perplexity, Crawlers & user agents", "summary": "Perplexity's user-initiated fetcher. It visits a page when a person asks Perplexity about it, and generally disregards robots.txt because it acts on a user request.", "detail": [ "Perplexity-User fetches pages when users ask Perplexity questions, so it can provide accurate answers and include links in responses.", "Perplexity documents that this fetcher generally disregards robots.txt rules, because it acts on a direct user request rather than as a background crawler.", "Like Perplexity's other agent, it does not collect training data, it serves user-facing answers only." ] }, { "slug": "claude-searchbot", "token": "Claude-SearchBot", "company": "Anthropic", "product": "Claude search", "category": "discovery", "robotsToken": "Claude-SearchBot", "affectsAiAnswers": "yes", "affectsSearch": null, "affectsTraining": false, "verdict": "allow-to-be-cited", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "docLabel": "Anthropic, Does Anthropic crawl the web?", "summary": "Anthropic's search crawler. It indexes content to improve Claude's search results, allow it to be eligible for citations in Claude search.", "detail": [ "Claude-SearchBot navigates the web to improve search result quality for Claude users. Anthropic states it \"analyzes online content specifically to enhance the relevance and accuracy of search responses.\"", "This is the Anthropic agent that matters for being surfaced and cited in Claude's search, not ClaudeBot, which is for model training.", "Anthropic publishes crawler IP ranges at claude.com/crawling/bots.json for verification." ] }, { "slug": "claudebot", "token": "ClaudeBot", "company": "Anthropic", "product": "Anthropic model training", "category": "training", "robotsToken": "ClaudeBot", "affectsAiAnswers": "no", "affectsSearch": null, "affectsTraining": true, "verdict": "your-choice-training", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "docLabel": "Anthropic, Does Anthropic crawl the web?", "summary": "Anthropic's training crawler. Blocking it opts your content out of training, it does NOT remove you from Claude's search answers.", "detail": [ "ClaudeBot is the crawler Anthropic uses to collect web content that could contribute to training its models. Anthropic states it \"helps enhance the utility and safety of our generative AI models by collecting web content that could potentially contribute to their training.\"", "ClaudeBot is a training control. Blocking it does not remove you from Claude's live search answers, that is governed by Claude-SearchBot.", "Treating ClaudeBot as Anthropic's citation crawler is a common error; the citation-relevant agent is Claude-SearchBot." ] }, { "slug": "claude-user", "token": "Claude-User", "company": "Anthropic", "product": "Claude (user-initiated)", "category": "user", "robotsToken": "Claude-User", "affectsAiAnswers": "indirect", "affectsSearch": null, "affectsTraining": false, "verdict": "user-initiated", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "docLabel": "Anthropic, Does Anthropic crawl the web?", "summary": "Anthropic's user-initiated fetcher. It visits a page when a person asks Claude about it.", "detail": [ "Claude-User supports Claude users: when individuals ask Claude questions, it may access websites using a Claude-User agent to answer them.", "It fetches on demand in response to a user's request, rather than crawling in the background, and does not gather training data.", "Allowing Claude-User lets Claude read a page a user points it at; blocking it prevents that on-demand fetch." ] }, { "slug": "ccbot", "token": "CCBot", "company": "Common Crawl", "product": "Common Crawl open corpus", "category": "training", "robotsToken": "CCBot", "affectsAiAnswers": "no", "affectsSearch": null, "affectsTraining": true, "verdict": "your-choice-training", "confidence": "documented", "lastVerified": "2026-09-04", "docUrl": "https://commoncrawl.org/ccbot", "docLabel": "Common Crawl, CCBot", "summary": "The crawler for Common Crawl's open web archive. Its data is a widely used training corpus, so blocking it is a training choice, not a live-answer one.", "detail": [ "CCBot is operated by Common Crawl, a nonprofit that maintains an open repository of web crawl data for research and development. Its user-agent is \"CCBot/2.0 (https://commoncrawl.org/faq/).\"", "Common Crawl's archive is one of the most widely used sources of AI training data, so allowing or blocking CCBot is effectively a decision about your content appearing in training corpora. It does not decide live AI-answer eligibility.", "Common Crawl warns that crawlers sometimes falsely identify as CCBot and recommends verifying with reverse DNS against its published IP ranges." ] }, { "slug": "bingbot", "token": "bingbot", "company": "Microsoft", "product": "Bing Search + Copilot", "category": "discovery", "robotsToken": "bingbot", "affectsAiAnswers": "yes", "affectsSearch": true, "affectsTraining": false, "verdict": "allow-to-be-cited", "confidence": "observed", "lastVerified": "2026-09-04", "docUrl": "https://www.bing.com/bingbot.htm", "docLabel": "Microsoft, Bingbot", "summary": "Microsoft's Bing search crawler. Microsoft Copilot is grounded in Bing web results, so Bingbot access affects both Bing and Copilot.", "detail": [ "Bingbot is Microsoft's crawler for Bing Search; it crawls and indexes pages for the Bing index. Its user-agent is \"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm).\"", "Microsoft Copilot is grounded in Bing web search, so being indexed by Bingbot is what makes a page reachable for Copilot's cited answers. We label the Copilot linkage \"observed\" because Microsoft documents Bingbot for Search, while the Copilot grounding relationship is stated in Microsoft's product messaging rather than a single crawler spec.", "Blocking Bingbot removes you from Bing Search and, by extension, from the results Copilot draws on." ] } ] }