{
  "name": "resulthack AI crawler directory",
  "description": "AI crawlers and assistant fetchers: who runs each one, what it is for, whether the operator says it obeys robots.txt, how to verify it, and the exact robots.txt token. Every fact is taken from the operator's own documentation, fetched with a plain GET on the checked date. A fact the operator does not state is the literal string \"unconfirmed\".",
  "homepage": "https://resulthack.com/",
  "updated": "2026-09-22",
  "unconfirmed": "The value \"unconfirmed\" means the operator's own documentation does not state this fact, or its page refused our fetch. It is never filled in from third-party lists or from memory.",
  "fields": {
    "token": "The product token for a robots.txt User-agent line, spelled as the operator documents it (robots.txt matching of the token is case-insensitive under RFC 9309).",
    "tokenDocumented": "true when the operator's own documentation names this token.",
    "operator": "Who runs the crawler, as stated by that organisation.",
    "purpose": "Array drawn from training, search, user-fetch; more than one entry is a mix the operator documents. \"unconfirmed\" when the operator documents none of these.",
    "purposeNote": "Optional: the operator's own wording where the purpose is a mix or does not fit the three categories.",
    "does": "One plain sentence on what it does.",
    "robotsTxt": "obeys: yes | not always | no | unconfirmed, plus the operator's documented detail and exceptions.",
    "userAgent": "What to look for in server logs.",
    "verification": "How the operator says to verify a request really comes from it, with its published IP list if any. via: set when the token has no traffic of its own and the list belongs to the crawler that fetches for it.",
    "blockEffect": "What the operator says blocking it does and does not change.",
    "evidence": "Verbatim sentences from the operator's documentation, each pointing at the source index that contains it, and the facts it backs.",
    "sources": "Operator documentation fetched with a plain GET, with the date checked."
  },
  "crawlers": [
    {
      "slug": "gptbot",
      "token": "GPTBot",
      "tokenDocumented": true,
      "operator": "OpenAI",
      "purpose": ["training"],
      "does": "Crawls public web pages whose content may be used to train OpenAI's generative AI foundation models.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "OpenAI documents GPTBot as a robots.txt token: disallowing it tells OpenAI the site's content should not be used for training. When it fetches robots.txt it may add a robots.txt marker to its user-agent string."
      },
      "userAgent": "The user-agent string contains \"GPTBot/1.4\"; OpenAI notes the version number may change.",
      "verification": {
        "method": "Match the request's IP address against OpenAI's published GPTBot IP list.",
        "ipRanges": { "url": "https://openai.com/gptbot.json", "ipv4": 18, "ipv6": 0, "listCreated": "2026-09-22" }
      },
      "blockEffect": "Blocking GPTBot only opts your content out of training. It does not remove you from ChatGPT search, which is controlled separately by OAI-SearchBot. If you allow both, OpenAI may use one crawl for both purposes.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "GPTBot is used to make our generative AI foundation models more useful and safe. It is used to crawl content that may be used in training our generative AI foundation models." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 0, "quote": "Disallowing GPTBot indicates a site’s content should not be used in training generative AI foundation models." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Each setting is independent of the others – for example, a webmaster can allow OAI-SearchBot in order to appear in search results while disallowing GPTBot to indicate that crawled content should not be used for training OpenAI’s generative AI foundation models." },
        { "facts": ["blockEffect"], "source": 0, "quote": "If your site has allowed both bots, we may use the results from just one crawl for both use cases to avoid duplicative crawling." },
        { "facts": ["robotsTxt", "userAgent"], "source": 0, "quote": "When fetching robots.txt files, we may add a robots.txt marker to the user-agent string to help site owners distinguish those requests from requests for other resources, especially when logs do not include paths." },
        { "facts": ["userAgent"], "source": 0, "quote": "Example user-agent string (the version number may change): Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4;" },
        { "facts": ["verification"], "source": 0, "quote": "Published IP addresses:" }
      ],
      "sources": [
        { "title": "OpenAI: Overview of OpenAI Crawlers", "url": "https://developers.openai.com/api/docs/bots", "checked": "2026-09-22" },
        { "title": "OpenAI: GPTBot IP ranges (gptbot.json)", "url": "https://openai.com/gptbot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "oai-searchbot",
      "token": "OAI-SearchBot",
      "tokenDocumented": true,
      "operator": "OpenAI",
      "purpose": ["search"],
      "does": "Crawls the web so that pages can be shown as results in ChatGPT's search features.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "OpenAI documents OAI-SearchBot as a robots.txt token and says a robots.txt change can take about 24 hours to reach its search systems."
      },
      "userAgent": "The user-agent string contains \"OAI-SearchBot/1.4\" (the version number may change); requests for robots.txt may carry an extra \"robots.txt\" marker.",
      "verification": {
        "method": "Match the request's IP address against OpenAI's published OAI-SearchBot IP list. OpenAI recommends allowing requests from these ranges if you want to appear in ChatGPT search.",
        "ipRanges": { "url": "https://openai.com/searchbot.json", "ipv4": 39, "ipv6": 0, "listCreated": "2026-01-02" }
      },
      "blockEffect": "Blocking OAI-SearchBot keeps your pages out of ChatGPT search answers, though OpenAI says they can still appear as navigational links. It does not decide whether your content is used for training; that is GPTBot.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "OAI-SearchBot is for search. OAI-SearchBot is used to surface websites in search results in ChatGPT’s search features." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though can still appear as navigational links." },
        { "facts": ["verification"], "source": 0, "quote": "To help ensure your site appears in search results, we recommend allowing OAI-SearchBot in your site’s robots.txt file and allowing requests from our published IP ranges below." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "For search results, please note it can take ~24 hours from a site’s robots.txt update for our systems to adjust." },
        { "facts": ["userAgent"], "source": 0, "quote": "Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; robots.txt;" },
        { "facts": ["blockEffect"], "source": 0, "quote": "Each setting is independent of the others" }
      ],
      "sources": [
        { "title": "OpenAI: Overview of OpenAI Crawlers", "url": "https://developers.openai.com/api/docs/bots", "checked": "2026-09-22" },
        { "title": "OpenAI: OAI-SearchBot IP ranges (searchbot.json)", "url": "https://openai.com/searchbot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "chatgpt-user",
      "token": "ChatGPT-User",
      "tokenDocumented": true,
      "operator": "OpenAI",
      "purpose": ["user-fetch"],
      "does": "Visits a web page when a ChatGPT or Custom GPT user's request needs it; it does not crawl the web automatically.",
      "robotsTxt": {
        "obeys": "not always",
        "detail": "OpenAI says that because these visits are initiated by a user, robots.txt rules may not apply."
      },
      "userAgent": "The user-agent string contains \"ChatGPT-User/1.0\".",
      "verification": {
        "method": "Match the request's IP address against OpenAI's published ChatGPT-User IP list.",
        "ipRanges": { "url": "https://openai.com/chatgpt-user.json", "ipv4": 229, "ipv6": 0, "listCreated": "2026-09-22" }
      },
      "blockEffect": "A Disallow line may not stop ChatGPT-User, because OpenAI says robots.txt may not apply to user-initiated visits. It also has no bearing on ChatGPT search: OpenAI says to use OAI-SearchBot for search opt-outs.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "When users ask ChatGPT or a CustomGPT a question, it may visit a web page with a ChatGPT-User agent." },
        { "facts": ["does"], "source": 0, "quote": "ChatGPT-User is not used for crawling the web in an automatic fashion." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 0, "quote": "Because these actions are initiated by a user, robots.txt rules may not apply." },
        { "facts": ["blockEffect"], "source": 0, "quote": "ChatGPT-User is not used to determine whether content may appear in Search. Please use OAI-SearchBot in robots.txt for managing Search opt outs and automatic crawl." },
        { "facts": ["userAgent"], "source": 0, "quote": "Full user-agent string: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0;" },
        { "facts": ["verification"], "source": 0, "quote": "Published IP addresses:" }
      ],
      "sources": [
        { "title": "OpenAI: Overview of OpenAI Crawlers", "url": "https://developers.openai.com/api/docs/bots", "checked": "2026-09-22" },
        { "title": "OpenAI: ChatGPT-User IP ranges (chatgpt-user.json)", "url": "https://openai.com/chatgpt-user.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "claudebot",
      "token": "ClaudeBot",
      "tokenDocumented": true,
      "operator": "Anthropic",
      "purpose": ["training"],
      "does": "Collects public web content that could contribute to training Anthropic's generative AI models.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Anthropic says its bots honour robots.txt directives and support the non-standard Crawl-delay extension. It says blocking by IP address may not work reliably, because that stops it from reading your robots.txt."
      },
      "userAgent": "unconfirmed",
      "verification": {
        "method": "Anthropic says a crawler whose source IP address is on its published list is coming from Anthropic.",
        "ipRanges": { "url": "https://claude.com/crawling/bots.json", "ipv4": 26, "ipv6": 0, "listCreated": "2026-08-18" }
      },
      "blockEffect": "Blocking ClaudeBot tells Anthropic to exclude your site's future content from its training datasets. Search and user fetches are controlled separately by Claude-SearchBot and Claude-User.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "ClaudeBot helps enhance the utility and safety of our generative AI models by collecting web content that could potentially contribute to their training." },
        { "facts": ["blockEffect"], "source": 0, "quote": "When a site restricts ClaudeBot access, it signals that the site's future materials should be excluded from our AI model training datasets." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "To limit crawling activity, we support the non-standard Crawl-delay extension to robots.txt." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Alternate methods like blocking IP address(es) from which Anthropic Bots operates may not work correctly or persistently guarantee an opt-out, as doing so impedes our ability to read your robots.txt file." },
        { "facts": ["verification"], "source": 0, "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic." }
      ],
      "sources": [
        { "title": "Claude Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "checked": "2026-09-22" },
        { "title": "Anthropic: crawler IP ranges (bots.json)", "url": "https://claude.com/crawling/bots.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "claude-searchbot",
      "token": "Claude-SearchBot",
      "tokenDocumented": true,
      "operator": "Anthropic",
      "purpose": ["search"],
      "does": "Crawls and analyses web content to improve the relevance and accuracy of search results for Claude users.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Anthropic says its bots honour robots.txt directives and support the non-standard Crawl-delay extension."
      },
      "userAgent": "unconfirmed",
      "verification": {
        "method": "Anthropic says a crawler whose source IP address is on its published list is coming from Anthropic.",
        "ipRanges": { "url": "https://claude.com/crawling/bots.json", "ipv4": 26, "ipv6": 0, "listCreated": "2026-08-18" }
      },
      "blockEffect": "Blocking Claude-SearchBot stops Anthropic indexing your content for search, which Anthropic says may reduce your visibility and accuracy in Claude's search results. It does not decide training use; that is ClaudeBot.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Disabling Claude-SearchBot on your site prevents our system from indexing your content for search optimization, which may reduce your site's visibility and accuracy in user search results." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "To limit crawling activity, we support the non-standard Crawl-delay extension to robots.txt." },
        { "facts": ["verification"], "source": 0, "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic." }
      ],
      "sources": [
        { "title": "Claude Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "checked": "2026-09-22" },
        { "title": "Anthropic: crawler IP ranges (bots.json)", "url": "https://claude.com/crawling/bots.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "claude-user",
      "token": "Claude-User",
      "tokenDocumented": true,
      "operator": "Anthropic",
      "purpose": ["user-fetch"],
      "does": "Fetches a web page when a Claude user's question needs it.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Unlike some user-triggered fetchers, Anthropic says Claude-User lets site owners control which sites these user-initiated requests can reach, and that its bots honour robots.txt directives."
      },
      "userAgent": "unconfirmed",
      "verification": {
        "method": "Anthropic says a crawler whose source IP address is on its published list is coming from Anthropic.",
        "ipRanges": { "url": "https://claude.com/crawling/bots.json", "ipv4": 26, "ipv6": 0, "listCreated": "2026-08-18" }
      },
      "blockEffect": "Blocking Claude-User stops Claude retrieving your pages in answer to a user's question, which Anthropic says may reduce your visibility for user-directed web search.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 0, "quote": "Claude-User allows site owners to control which sites can be accessed through these user-initiated requests. Disabling Claude-User on your site prevents our system from retrieving your content in response to a user query, which may reduce your site's visibility for user-directed web search." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt." },
        { "facts": ["verification"], "source": 0, "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic." }
      ],
      "sources": [
        { "title": "Claude Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "checked": "2026-09-22" },
        { "title": "Anthropic: crawler IP ranges (bots.json)", "url": "https://claude.com/crawling/bots.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "perplexitybot",
      "token": "PerplexityBot",
      "tokenDocumented": true,
      "operator": "Perplexity",
      "purpose": ["search"],
      "does": "Crawls the web to surface and link websites in Perplexity's search results; Perplexity says it is not used to crawl content for AI foundation models.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Perplexity documents PerplexityBot as a robots.txt token and says changes can take up to 24 hours to take effect."
      },
      "userAgent": "The user-agent string contains \"PerplexityBot/1.0\".",
      "verification": {
        "method": "Match the request's IP address against Perplexity's published PerplexityBot IP list. For firewalls, Perplexity suggests rules that combine the user-agent with those IP ranges.",
        "ipRanges": { "url": "https://www.perplexity.com/perplexitybot.json", "ipv4": 8, "ipv6": 0, "listCreated": "2025-02-07" }
      },
      "blockEffect": "Blocking PerplexityBot keeps your site out of Perplexity's search results, since Perplexity recommends allowing it to appear there. Perplexity says PerplexityBot does not collect content for AI foundation models, so there is no training opt-out to gain from blocking it.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does", "blockEffect"], "source": 0, "quote": "is designed to surface and link websites in search results on Perplexity. It is not used to crawl content for AI foundation models." },
        { "facts": ["blockEffect", "verification"], "source": 0, "quote": "To ensure your site appears in search results, we recommend allowing PerplexityBot in your site’s robots.txt file and permitting requests from our published IP ranges listed below." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Webmasters can use the following robots.txt tags to manage how their sites and content interact with Perplexity. Each setting works independently, and it may take up to 24 hours for our systems to reflect changes." },
        { "facts": ["verification"], "source": 0, "quote": "Set up a rule that combines both User-Agent and IP address conditions:" },
        { "facts": ["userAgent"], "source": 0, "quote": "Full user-agent string: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0;" }
      ],
      "sources": [
        { "title": "Perplexity: Perplexity Crawlers", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "checked": "2026-09-22" },
        { "title": "Perplexity: PerplexityBot IP ranges (perplexitybot.json)", "url": "https://www.perplexity.com/perplexitybot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "perplexity-user",
      "token": "Perplexity-User",
      "tokenDocumented": true,
      "operator": "Perplexity",
      "purpose": ["user-fetch"],
      "does": "Visits a web page when a Perplexity user's question needs it, and may link that page in the answer; Perplexity says it is not used for crawling or for training AI models.",
      "robotsTxt": {
        "obeys": "not always",
        "detail": "Perplexity says that because a user requested the fetch, Perplexity-User generally ignores robots.txt rules."
      },
      "userAgent": "The user-agent string contains \"Perplexity-User/1.0\".",
      "verification": {
        "method": "Match the request's IP address against Perplexity's published Perplexity-User IP list.",
        "ipRanges": { "url": "https://www.perplexity.com/perplexity-user.json", "ipv4": 4, "ipv6": 0, "listCreated": "2025-10-17" }
      },
      "blockEffect": "A Disallow line will generally not stop Perplexity-User, because Perplexity says it generally ignores robots.txt. Its published IP list is how the operator lets you identify it, for example in a firewall rule.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "supports user actions within Perplexity. When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a link to the page in its response." },
        { "facts": ["does"], "source": 0, "quote": "It is not used for web crawling or to collect content for training AI foundation models." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 0, "quote": "Since a user requested the fetch, this fetcher generally ignores robots.txt rules." },
        { "facts": ["verification", "blockEffect"], "source": 0, "quote": "Always use the most current IP ranges from the official JSON endpoints. These addresses are updated regularly and should be the source of truth for your WAF configurations." },
        { "facts": ["userAgent"], "source": 0, "quote": "Full user-agent string: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0;" }
      ],
      "sources": [
        { "title": "Perplexity: Perplexity Crawlers", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "checked": "2026-09-22" },
        { "title": "Perplexity: Perplexity-User IP ranges (perplexity-user.json)", "url": "https://www.perplexity.com/perplexity-user.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "google-extended",
      "token": "Google-Extended",
      "tokenDocumented": true,
      "operator": "Google",
      "purpose": ["training", "search"],
      "purposeNote": "Google says the token controls both training of future Gemini models and grounding: giving Gemini content from the Google Search index at prompt time (in Gemini Apps and Grounding with Google Search on Vertex AI).",
      "does": "A robots.txt control token, not a separate crawler: it decides whether content Google has already crawled may be used to train Gemini models and to ground Gemini's answers.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Google-Extended exists only as a robots.txt token; Google says its common crawlers always obey robots.txt rules when crawling automatically."
      },
      "userAgent": "None of its own: Google says there is no separate HTTP user-agent string for Google-Extended; crawling is done with Google's existing user agents.",
      "verification": {
        "via": "Google's common crawlers",
        "method": "There is no Google-Extended traffic to verify. For the Google crawlers that do the fetching, Google says to run a reverse DNS lookup on the IP, check the name ends in googlebot.com, google.com or googleusercontent.com, then run a forward lookup and confirm it returns the same IP; or match the IP against the published common-crawlers list.",
        "ipRanges": { "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "ipv4": 170, "ipv6": 147, "listCreated": "2026-09-22" }
      },
      "blockEffect": "Blocking Google-Extended opts your content out of Gemini training and Gemini grounding. Google says it does not affect your inclusion or ranking in Google Search.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "purposeNote", "does"], "source": 0, "quote": "is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models that power Gemini Apps and Vertex AI API for Gemini and for grounding (providing content from the Google Search index to the model at prompt time to improve factuality and relevancy) in Gemini Apps and Grounding with Google Search on Vertex AI." },
        { "facts": ["userAgent", "does", "robotsTxt"], "source": 0, "quote": "Google-Extended doesn't have a separate HTTP request user agent string. Crawling is done with existing Google user agent strings; the robots.txt user-agent token is used in a control capacity." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Google-Extended does not impact a site's inclusion in Google Search nor is it used as a ranking signal in Google Search." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "They always obey robots.txt rules when crawling automatically." },
        { "facts": ["verification"], "source": 0, "quote": "The common crawlers generally crawl from the IP ranges published in the common-crawlers.json object, and the reverse DNS mask of their hostname matches crawl-***-***-***-***.googlebot.com or geo-crawl-***-***-***-***.geo.googlebot.com." },
        { "facts": ["verification"], "source": 1, "quote": "Verify that the domain name is either googlebot.com, google.com, or googleusercontent.com." },
        { "facts": ["verification"], "source": 1, "quote": "Verify that it's the same as the original accessing IP address from your logs." }
      ],
      "sources": [
        { "title": "Google: List of Google's common crawlers", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "checked": "2026-09-22" },
        { "title": "Google: Verify requests from Google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "checked": "2026-09-22" },
        { "title": "Google: common crawler IP ranges (common-crawlers.json)", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "applebot-extended",
      "token": "Applebot-Extended",
      "tokenDocumented": true,
      "operator": "Apple",
      "purpose": ["training"],
      "does": "A robots.txt control token, not a crawler: it decides whether pages Applebot has crawled may be used to train Apple's generative AI foundation models.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Applebot-Extended exists only as a robots.txt rule; Apple says it does not crawl webpages and is used only to decide how data crawled by Applebot is used."
      },
      "userAgent": "None of its own: Apple says Applebot-Extended does not crawl webpages. The fetching is done by Applebot.",
      "verification": {
        "via": "Applebot",
        "method": "There is no Applebot-Extended traffic to verify. Applebot itself is identified by reverse DNS in the applebot.apple.com domain, or by matching the IP against Apple's published CIDR list.",
        "ipRanges": { "url": "https://search.developer.apple.com/applebot.json", "ipv4": 24, "ipv6": 0, "listCreated": "2026-09-15" }
      },
      "blockEffect": "Disallowing Applebot-Extended opts out of training Apple's foundation models, but Apple says your pages can still appear in its search results and the rule is not a ranking factor. To keep content out of AI-generated \"broad world knowledge\" answers in Siri and Search, Apple points to the nosnippet robots meta tag instead.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does"], "source": 0, "quote": "With Applebot-Extended, web publishers can choose to opt out of their website content being used to train Apple’s general purpose foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools." },
        { "facts": ["robotsTxt", "userAgent", "does", "blockEffect"], "source": 0, "quote": "Applebot-Extended does not crawl webpages. Webpages that disallow Applebot-Extended can still be included in search results. Applebot-Extended is only used to determine how to use the data crawled by the Applebot user agent." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Site rules for Applebot-Extended are not considered in ranking for Search." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Web publishers can opt out of their content being used in these broad world knowledge answers by applying the nosnippet meta tag to specific content." },
        { "facts": ["verification"], "source": 0, "quote": "Traffic coming from Applebot is generally identified by using reverse DNS in the *. applebot.apple.com domain." },
        { "facts": ["verification"], "source": 0, "quote": "Another way is to match the IP address with a CIDR prefix contained in the following JSON file: Applebot IP CIDRs." }
      ],
      "sources": [
        { "title": "Apple Support: About Applebot", "url": "https://support.apple.com/en-us/119829", "checked": "2026-09-22" },
        { "title": "Apple: Applebot IP CIDRs (applebot.json)", "url": "https://search.developer.apple.com/applebot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "applebot",
      "token": "Applebot",
      "tokenDocumented": true,
      "operator": "Apple",
      "purpose": ["search", "training"],
      "purposeNote": "Apple says Applebot's data powers search in Spotlight, Siri and Safari, may help train Apple's foundation models (opt out with Applebot-Extended), and may be used as context for AI-generated answers (opt out with nosnippet).",
      "does": "Crawls and renders web pages for search in Apple's products such as Spotlight, Siri and Safari; the same crawl may also feed Apple's AI features.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Apple says Applebot respects robots.txt in general search crawls aimed at it. If your robots.txt does not mention Applebot but does mention Googlebot, Applebot follows the Googlebot rules. It does not follow Crawl-delay. The separate iTMS user agent (Apple Podcasts) does not follow robots.txt."
      },
      "userAgent": "The user-agent string contains \"Applebot/\" followed by a version (for example Applebot/0.1).",
      "verification": {
        "method": "Apple says Applebot traffic is identified by reverse DNS in the applebot.apple.com domain (check with the host command both ways), or by matching the IP against Apple's published CIDR list.",
        "ipRanges": { "url": "https://search.developer.apple.com/applebot.json", "ipv4": 24, "ipv6": 0, "listCreated": "2026-09-15" }
      },
      "blockEffect": "Blocking Applebot removes your pages from search in Spotlight, Siri and Safari. If you only want to opt out of AI training, Apple's documented route is to disallow Applebot-Extended and keep Applebot allowed.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "purposeNote", "does", "blockEffect"], "source": 0, "quote": "The data crawled by Applebot is used to power various features, such as the search technology integrated into many user experiences in Appleʼs ecosystem including Spotlight, Siri, and Safari. Enabling Applebot in robots.txt allows website content to appear in search results for Apple users around the world in these products." },
        { "facts": ["purpose", "purposeNote", "blockEffect"], "source": 0, "quote": "The data crawled by Applebot may also be used to help train Apple foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools. Web publishers can opt-out from having their content used to train generative foundation models by disallowing Applebot-Extended in the robots.txt file." },
        { "facts": ["purposeNote"], "source": 0, "quote": "Applebot crawled data may be used to provide additional context and up-to-date content when AI models are used to generate output for display in Apple products and services." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "If robots instructions don't mention Applebot but mention Googlebot, the Apple robot will follow Googlebot instructions. Applebot does not follow crawl-delay." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "The iTMS user agent does not follow robots.txt, as it is not a general search crawler." },
        { "facts": ["userAgent"], "source": 0, "quote": "The user-agent string contains ”Applebot” and other information." },
        { "facts": ["verification"], "source": 0, "quote": "Traffic coming from Applebot is generally identified by using reverse DNS in the *. applebot.apple.com domain." },
        { "facts": ["verification"], "source": 0, "quote": "Another way is to match the IP address with a CIDR prefix contained in the following JSON file: Applebot IP CIDRs." }
      ],
      "sources": [
        { "title": "Apple Support: About Applebot", "url": "https://support.apple.com/en-us/119829", "checked": "2026-09-22" },
        { "title": "Apple: Applebot IP CIDRs (applebot.json)", "url": "https://search.developer.apple.com/applebot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "ccbot",
      "token": "CCBot",
      "tokenDocumented": true,
      "operator": "Common Crawl",
      "purpose": "unconfirmed",
      "purposeNote": "Common Crawl describes its purpose as a free, open copy of the web for research and analysis by anyone. Its pages we checked do not say whether the archive is used to train AI models, so we do not file it under training, search or user fetch.",
      "does": "Crawls a sample of the web for Common Crawl's free, openly downloadable archive of web pages.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Common Crawl says CCBot checks robots.txt before fetching, obeys Crawl-delay, honours link-level nofollow and keeps re-checking robots.txt for changes."
      },
      "userAgent": "The user-agent string contains \"CCBot/2.0\". Common Crawl warns that other crawlers falsely identify themselves as CCBot.",
      "verification": {
        "method": "CCBot runs on dedicated IP ranges with reverse DNS in crawl.commoncrawl.org (not yet over IPv6); check with a reverse and forward lookup, or match the IP against the published list.",
        "ipRanges": { "url": "https://index.commoncrawl.org/ccbot.json", "ipv4": 4, "ipv6": 1, "listCreated": "2026-08-11" }
      },
      "blockEffect": "Common Crawl says CCBot stops crawling your site once robots.txt disallows it. Its pages we checked do not say whether pages already in the archive are removed; it separately runs an opt-out registry.",
      "evidence": [
        { "facts": ["token", "operator", "robotsTxt", "blockEffect"], "source": 0, "quote": "To prevent Common Crawl from crawling your website, include the following in your robots.txt : User-agent: CCBot Disallow: /" },
        { "facts": ["operator", "purposeNote"], "source": 1, "quote": "Common Crawl is a 501(c)(3) non-profit organization dedicated to providing a copy of the Internet to Internet researchers, companies and individuals at no cost for the purpose of research and analysis." },
        { "facts": ["does"], "source": 1, "quote": "Common Crawl's dataset is a sample of the web, and we do not generally archive any entire website but a randomly selected subset of it." },
        { "facts": ["robotsTxt"], "source": 1, "quote": "CCBot is an automated crawler, checking first the robots.txt , and if crawling a page is allowed, fetches pages using HTTP GET requests." },
        { "facts": ["robotsTxt"], "source": 1, "quote": "We obey the Crawl-delay parameter for robots.txt ." },
        { "facts": ["robotsTxt"], "source": 1, "quote": "We currently honor the nofollow attribute as it applies to links embedded on your site." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 1, "quote": "Add these lines to your robots.txt file and our crawler will stop crawling your website:" },
        { "facts": ["robotsTxt"], "source": 1, "quote": "We will periodically continue to check if the robots.txt file has been updated." },
        { "facts": ["blockEffect"], "source": 1, "quote": "You may also wish to be added to our opt-out registry." },
        { "facts": ["userAgent"], "source": 0, "quote": "Please note that we are aware of crawlers falsely identifying themselves as CCBot. We recommend verifying UserAgent strings to ensure authenticity." },
        { "facts": ["verification"], "source": 0, "quote": "CCBot is now run on dedicated IP address ranges with reverse DNS (except over IPv6 where reverse DNS is not yet supported.)" }
      ],
      "sources": [
        { "title": "Common Crawl: CCBot", "url": "https://commoncrawl.org/ccbot", "checked": "2026-09-22" },
        { "title": "Common Crawl: FAQ", "url": "https://commoncrawl.org/faq", "checked": "2026-09-22" },
        { "title": "Common Crawl: CCBot IP ranges (ccbot.json)", "url": "https://index.commoncrawl.org/ccbot.json", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "bytespider",
      "token": "Bytespider",
      "tokenDocumented": false,
      "operator": "unconfirmed",
      "purpose": "unconfirmed",
      "does": "unconfirmed",
      "robotsTxt": { "obeys": "unconfirmed", "detail": "unconfirmed" },
      "userAgent": "unconfirmed",
      "verification": "unconfirmed",
      "blockEffect": "unconfirmed",
      "evidence": [],
      "sources": [],
      "searched": [
        { "url": "https://www.bytedance.com/en/", "checked": "2026-09-22", "result": "Page loaded; no mention of Bytespider." },
        { "url": "https://www.bytedance.com/robots.txt", "checked": "2026-09-22", "result": "Loaded; no mention of Bytespider." },
        { "url": "https://www.bytedance.com/en/bytespider", "checked": "2026-09-22", "result": "Page not found (the site's 404 page)." }
      ]
    },
    {
      "slug": "meta-externalagent",
      "token": "Meta-ExternalAgent",
      "tokenDocumented": true,
      "operator": "Meta",
      "purpose": ["training"],
      "purposeNote": "Meta's wording is a mix: \"training foundation AI models or improving products by indexing content directly\".",
      "does": "Crawls the web for uses such as training Meta's foundation AI models or improving Meta products by indexing content directly.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Meta says to block its crawlers with a Disallow for the relevant crawler in robots.txt, and that robots.txt may be cached for up to 24 hours. Meta's own example writes the token in lower case (meta-externalagent)."
      },
      "userAgent": "The user-agent string contains \"meta-externalagent/1.1\".",
      "verification": "unconfirmed",
      "blockEffect": "unconfirmed",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "purposeNote", "does"], "source": 0, "quote": "The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models or improving products by indexing content directly." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Please allow up to 24 hours for changes to `robots.txt` to take effect because crawlers may cache the contents of `robots.txt` for up to 24 hours." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "User-agent: meta-externalagent" },
        { "facts": ["userAgent"], "source": 0, "quote": "`meta-externalagent/1.1`" }
      ],
      "sources": [
        { "title": "Meta for Developers: Meta Web Crawlers", "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "meta-webindexer",
      "token": "Meta-WebIndexer",
      "tokenDocumented": true,
      "operator": "Meta",
      "purpose": ["search"],
      "does": "Crawls the web to improve Meta AI's search results so Meta AI can cite and link to pages in its answers.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Meta says to block its crawlers with a Disallow for the relevant crawler in robots.txt; the documented exceptions are Meta-ExternalFetcher and FacebookExternalHit, not this crawler."
      },
      "userAgent": "The user-agent string contains \"meta-webindexer/1.1\".",
      "verification": "unconfirmed",
      "blockEffect": "Meta says allowing Meta-WebIndexer helps it cite and link to your content in Meta AI's responses, so blocking it gives that up.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does", "blockEffect"], "source": 0, "quote": "The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt. The Meta-ExternalFetcher crawler may bypass robots.txt because it performs fetches that were requested by the user. Also, the FacebookExternalHit crawler might bypass robots.txt when performing security or integrity checks." },
        { "facts": ["userAgent"], "source": 0, "quote": "`meta-webindexer/1.1`" }
      ],
      "sources": [
        { "title": "Meta for Developers: Meta Web Crawlers", "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "meta-externalfetcher",
      "token": "Meta-ExternalFetcher",
      "tokenDocumented": true,
      "operator": "Meta",
      "purpose": ["user-fetch"],
      "does": "Fetches individual links at a user's request, including helping Meta's AI navigate websites to complete tasks for users.",
      "robotsTxt": {
        "obeys": "not always",
        "detail": "Meta says this crawler may bypass robots.txt because it performs fetches that were requested by the user."
      },
      "userAgent": "The user-agent string contains \"meta-externalfetcher/1.1\".",
      "verification": "unconfirmed",
      "blockEffect": "A Disallow line may not stop Meta-ExternalFetcher, because Meta says it may bypass robots.txt for user-requested fetches.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does", "robotsTxt", "blockEffect"], "source": 0, "quote": "The Meta-ExternalFetcher crawler fetches individual links at a user's request and supports product functions such as evaluating and improving agentic AI capabilities—including helping AI navigate websites to complete tasks for users. Accordingly, this crawler may bypass robots.txt rules." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "The Meta-ExternalFetcher crawler may bypass robots.txt because it performs fetches that were requested by the user." },
        { "facts": ["userAgent"], "source": 0, "quote": "`meta-externalfetcher/1.1`" }
      ],
      "sources": [
        { "title": "Meta for Developers: Meta Web Crawlers", "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "amazonbot",
      "token": "Amazonbot",
      "tokenDocumented": true,
      "operator": "Amazon",
      "purpose": ["training"],
      "purposeNote": "Amazon's wording is a mix: Amazonbot is \"used to improve our products and services\" and \"may be used to train Amazon AI models\".",
      "does": "Crawls the web to improve Amazon's products and services; the content may be used to train Amazon AI models.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Amazon says automated crawling honours robots.txt user-agent and allow/disallow rules, reads robots.txt per host or uses a copy cached for up to 30 days, and behaves as if there is no robots.txt when the file cannot be fetched. It does not support Crawl-delay. It also honours rel=nofollow and the robots meta tags noarchive (which Amazon says means do not use the page for model training), noindex and none."
      },
      "userAgent": "The user-agent string contains \"Amazonbot/0.1\".",
      "verification": {
        "method": "Match the request's IP address against Amazon's published Amazonbot IP list.",
        "ipRanges": { "url": "https://developer.amazon.com/amazonbot/ip-addresses/", "ipv4": 1292, "ipv6": 0, "listCreated": "2026-09-08" }
      },
      "blockEffect": "Blocking Amazonbot stops this crawler; Amazon's search experiences (such as Alexa) are controlled separately by Amzn-SearchBot. To stay crawlable but keep a single page out of model training, Amazon documents the noarchive robots meta tag.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "purposeNote", "does"], "source": 0, "quote": "Amazonbot is used to improve our products and services. This helps us provide more accurate information to customers and may be used to train Amazon AI models." },
        { "facts": ["blockEffect"], "source": 0, "quote": "Each user agent setting is independent of the others, and may take ~24 hours for our systems to reflect changes." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol , honoring the user-agent and the allow/disallow directives. They will fetch host-level robots.txt files or use a cached copy from the last 30 days. When a file can’t be fetched, they will behave as if it does not exist." },
        { "facts": ["robotsTxt", "blockEffect"], "source": 0, "quote": "When these user agents access web pages they respect the link-level rel=nofollow directive, and page level robots meta tags of noarchive (do not use the page for model training), noindex (do not index the page) and none (do not index the page). They do not support the crawl-delay directive." },
        { "facts": ["userAgent"], "source": 0, "quote": "Example User Agent String: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36" },
        { "facts": ["verification"], "source": 0, "quote": "Published IP Addresses:" }
      ],
      "sources": [
        { "title": "Amazon Developer: About Amazonbot", "url": "https://developer.amazon.com/amazonbot", "checked": "2026-09-22" },
        { "title": "Amazon Developer: Amazonbot IP addresses", "url": "https://developer.amazon.com/amazonbot/ip-addresses/", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "amzn-searchbot",
      "token": "Amzn-SearchBot",
      "tokenDocumented": true,
      "operator": "Amazon",
      "purpose": ["search"],
      "does": "Crawls the web for search experiences in Amazon products such as Alexa; Amazon says it does not crawl content for generative AI model training.",
      "robotsTxt": {
        "obeys": "yes",
        "detail": "Amazon says it honours robots.txt, and that if your robots.txt does not mention Amzn-SearchBot but allows other search bots, it follows the rules given to those other search bots. It does not support Crawl-delay."
      },
      "userAgent": "The user-agent string contains \"Amzn-SearchBot/0.1\".",
      "verification": {
        "method": "Match the request's IP address against Amazon's published Amzn-SearchBot IP list.",
        "ipRanges": { "url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "ipv4": 816, "ipv6": 0, "listCreated": "2026-09-08" }
      },
      "blockEffect": "Amazon says allowing Amzn-SearchBot makes your content eligible for search experiences such as Alexa, so blocking it gives that up. It has no training role to opt out of.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does", "blockEffect", "robotsTxt"], "source": 0, "quote": "Amzn-SearchBot is used to improve search experiences in Amazon products and services. By permitting Amzn-SearchBot access to your website, your content is eligible to appear in search experiences such as Alexa. If robots.txt files don’t mention Amzn-SearchBot but allow other search bots, Amzn-SearchBot will crawl in accordance with the robots.txt directives given to other search bots. Amzn-SearchBot does not crawl content for generative AI model training." },
        { "facts": ["robotsTxt"], "source": 0, "quote": "They do not support the crawl-delay directive." },
        { "facts": ["userAgent"], "source": 0, "quote": "Example User Agent String: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36" },
        { "facts": ["verification"], "source": 0, "quote": "Published IP Addresses:" }
      ],
      "sources": [
        { "title": "Amazon Developer: About Amazonbot", "url": "https://developer.amazon.com/amazonbot", "checked": "2026-09-22" },
        { "title": "Amazon Developer: Amzn-SearchBot IP addresses", "url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "checked": "2026-09-22" }
      ]
    },
    {
      "slug": "amzn-user",
      "token": "Amzn-User",
      "tokenDocumented": true,
      "operator": "Amazon",
      "purpose": ["user-fetch"],
      "does": "Fetches live information from the web on a user's behalf, for example to answer an Alexa question that needs up-to-date information.",
      "robotsTxt": {
        "obeys": "not always",
        "detail": "Amazon says that because Amzn-User's actions can be initiated by a user, it may not follow all robots.txt directives."
      },
      "userAgent": "The user-agent string contains \"Amzn-User/0.1\".",
      "verification": {
        "method": "Match the request's IP address against Amazon's published Amzn-User IP list.",
        "ipRanges": { "url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "ipv4": 1023, "ipv6": 0, "listCreated": "2025-11-04" }
      },
      "blockEffect": "A Disallow line may not fully stop Amzn-User, because Amazon says it may not follow all robots.txt directives. Amazon says it does not crawl content for model training.",
      "evidence": [
        { "facts": ["token", "operator", "purpose", "does", "robotsTxt", "blockEffect"], "source": 0, "quote": "Amzn-User supports user actions, such as responding to Alexa queries that require up-to-date information. For example, when a customer asks a question, Amzn-User may fetch live information from the web to provide accurate answers on the user’s behalf. Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.Amzn-User does not crawl content for generative AI model training." },
        { "facts": ["userAgent"], "source": 0, "quote": "Example User Agent String: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36" },
        { "facts": ["verification"], "source": 0, "quote": "Published IP Addresses:" }
      ],
      "sources": [
        { "title": "Amazon Developer: About Amazonbot", "url": "https://developer.amazon.com/amazonbot", "checked": "2026-09-22" },
        { "title": "Amazon Developer: Amzn-User IP addresses", "url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "checked": "2026-09-22" }
      ]
    }
  ]
}