{
  "name": "resulthack /opt-out/: crawlers outside the documented-training group",
  "description": "Crawlers a site owner deciding on AI training may want to block, but whose operator does not document an AI-training purpose for them: either the operator documents a different purpose while others train on the data it collects (Common Crawl's CCBot), or the operator documents no purpose for the token at all. For each: the operator's own words quoted with the page and the date checked, or the statement that the operator documents no purpose; and what named third parties say, labelled as third-party, with the page and the date checked. Every page was fetched with a plain GET on the date given; tools/build-opt-out.cjs --verify-sources re-fetches them and reports any quote that is no longer on its page.",
  "homepage": "https://resulthack.com/opt-out/",
  "updated": "2026-09-26",
  "fields": {
    "token": "The robots.txt product token.",
    "operator": "Who runs it, as far as anyone documents. For a token its operator's crawler page no longer lists, the organisation the token is commonly attributed to.",
    "operatorSays.documentsPurpose": "true when the operator's own pages state what the crawler (or the data it collects) is for; false when they do not, in which case the page says 'the operator documents no purpose' with the page and date checked.",
    "operatorSays.quotes": "The operator's own sentences, verbatim, each with url and checked (YYYY-MM-DD). context says where on the page the sentence sits and what it is about.",
    "operatorSays.searched": "Operator pages checked that do not mention the token, with the date and what was found.",
    "thirdParty": "What a named third party says, verbatim, with url, the publication date when the page states one, and checked. Never the operator's statement; shown on the page under a 'third party' label.",
    "robotsNote": "Optional: something the operator documents about how it reads robots.txt that differs from plain RFC 9309 matching."
  },
  "crawlers": [
    {
      "token": "CCBot",
      "slug": "ccbot",
      "operator": "Common Crawl",
      "why": "Common Crawl documents its archive as a free copy of the web for research and analysis, not as AI training. Its own Terms of Use name training AI systems as one use others make of the archive, and third parties document that many large language models are trained on it.",
      "operatorSays": {
        "documentsPurpose": true,
        "quotes": [
          {
            "quote": "Common Crawl is a 501(c)(3) non-profit organization dedicated to providing a copy of the Internet to Internet researchers, companies and individuals at no cost for the purpose of research and analysis.",
            "context": "FAQ, answer to “What is Common Crawl?”",
            "url": "https://commoncrawl.org/faq",
            "checked": "2026-09-26"
          },
          {
            "quote": "People have used the data to improve language translation software, predict trends, track disease propagation, and much more.",
            "context": "FAQ, answer to “What can you do with Common Crawl data?”",
            "url": "https://commoncrawl.org/faq",
            "checked": "2026-09-26"
          },
          {
            "quote": "use of Crawled Content in connection with artificial intelligence, machine learning, or other similar technologies, including, without limitation, large language models and neural networks (collectively, \"AI Systems\"), including the use of Crawled Content for the purpose of developing, training, or deploying AI Systems",
            "context": "Terms of Use (last updated 7 March 2024), indemnification clause: whoever uses the archive covers Common Crawl's costs from claims arising from, among other things, this use. It is the only place on the pages checked where Common Crawl mentions AI training.",
            "url": "https://commoncrawl.org/terms-of-use",
            "checked": "2026-09-26"
          },
          {
            "quote": "Add these lines to your robots.txt file and our crawler will stop crawling your website: User-agent: CCBot Disallow: /",
            "context": "FAQ, answer to “How can I block the Common Crawl CCBot?”. The pages checked do not say whether blocking removes your pages from crawls already published; the same answer points to a separate opt-out registry.",
            "url": "https://commoncrawl.org/faq",
            "checked": "2026-09-26"
          }
        ]
      },
      "thirdParty": [
        {
          "who": "Mozilla Foundation",
          "title": "Training Data for the Price of a Sandwich: Common Crawl's Impact on Generative AI (Stefan Baack and Mozilla Insights)",
          "published": "2024-02-06",
          "quote": "This data has been valuable for many researchers, but since 2020 when OpenAI published GPT-3, the large language model (LLM) that still powers the free version of ChatGPT, Common Crawl has become one of the most important sources of training data for generative AI.",
          "url": "https://foundation.mozilla.org/en/research/library/generative-ai-training-data/common-crawl/",
          "checked": "2026-09-26"
        },
        {
          "who": "ai.robots.txt (community-maintained list of AI crawlers, MIT licence)",
          "title": "robots.json, the CCBot entry's “function” field",
          "quote": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.",
          "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json",
          "checked": "2026-09-26"
        },
        {
          "who": "Cloudflare",
          "title": "Declare your AIndependence: block AI bots, scrapers and crawlers with a single click",
          "published": "2024-07-03",
          "quote": "In the graph below, which looks at disallowed crawlers for these sites, we see that customers most often reference GPTBot, CCBot, and Google in robots.txt, but do not specifically disallow popular AI crawlers like Bytespider and ClaudeBot.",
          "url": "https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click/",
          "checked": "2026-09-26"
        }
      ]
    },
    {
      "token": "Bytespider",
      "slug": "bytespider",
      "operator": "ByteDance (attributed by third parties; ByteDance documents no crawler of this name)",
      "why": "No page of ByteDance's that we could find names Bytespider. Third parties attribute it to ByteDance and report it collects training data for ByteDance's language models.",
      "operatorSays": {
        "documentsPurpose": false,
        "url": "https://www.bytedance.com/en/",
        "checked": "2026-09-26",
        "searched": [
          { "url": "https://www.bytedance.com/en/", "checked": "2026-09-26", "result": "Loaded; no mention of Bytespider." },
          { "url": "https://www.bytedance.com/robots.txt", "checked": "2026-09-26", "result": "Loaded; three lines (User-agent: *, Allow: /, a Sitemap), no mention of Bytespider." },
          { "url": "https://www.bytedance.com/en/bytespider", "checked": "2026-09-26", "result": "The site's “not found” page." }
        ],
        "quotes": []
      },
      "thirdParty": [
        {
          "who": "Cloudflare",
          "title": "Declare your AIndependence: block AI bots, scrapers and crawlers with a single click",
          "published": "2024-07-03",
          "quote": "Operated by ByteDance, the Chinese company that owns TikTok, Bytespider is reportedly used to gather training data for its large language models (LLMs), including those that support its ChatGPT rival, Doubao.",
          "url": "https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click/",
          "checked": "2026-09-26"
        },
        {
          "who": "Cloudflare",
          "title": "Declare your AIndependence: block AI bots, scrapers and crawlers with a single click",
          "published": "2024-07-03",
          "quote": "Among the top AI bots that we see, Bytespider not only leads in terms of number of requests but also in both the extent of its Internet property crawling and the frequency with which it is blocked.",
          "url": "https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click/",
          "checked": "2026-09-26"
        },
        {
          "who": "ai.robots.txt (community-maintained list of AI crawlers, MIT licence)",
          "title": "robots.json, the Bytespider entry's “description” field (its “respect” field, on robots.txt, reads “No”)",
          "quote": "Downloads data to train LLMS, including ChatGPT competitors.",
          "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json",
          "checked": "2026-09-26"
        }
      ]
    },
    {
      "token": "cohere-ai",
      "slug": "cohere-ai",
      "operator": "Cohere (attributed by third parties; Cohere's crawler page does not name this token)",
      "why": "Cohere's crawler page does not name cohere-ai and says Cohere does not crawl the web to train its models. A third-party list describes cohere-ai as fetching pages for user prompts.",
      "operatorSays": {
        "documentsPurpose": false,
        "url": "https://docs.cohere.com/docs/cohere-web-crawlers",
        "checked": "2026-09-26",
        "searched": [
          { "url": "https://docs.cohere.com/docs/cohere-web-crawlers", "checked": "2026-09-26", "result": "Loaded; the crawler table reads “N/A”. No mention of cohere-ai." }
        ],
        "quotes": [
          {
            "quote": "We do not use Cohere bots or user agents for the purpose of crawling or scraping web content to train generative AI foundation models at this time.",
            "context": "Cohere Web Crawlers, first paragraph. About Cohere's bots in general; it does not name cohere-ai.",
            "url": "https://docs.cohere.com/docs/cohere-web-crawlers",
            "checked": "2026-09-26"
          },
          {
            "quote": "Should Cohere make use of these tools to crawl or scrape web content to train AI models in the future, Cohere will identify those bots and/or user agents in the table below alongside information about how to disallow their access to your site.",
            "context": "Cohere Web Crawlers, first paragraph. The table below it reads “N/A”.",
            "url": "https://docs.cohere.com/docs/cohere-web-crawlers",
            "checked": "2026-09-26"
          }
        ]
      },
      "thirdParty": [
        {
          "who": "ai.robots.txt (community-maintained list of AI crawlers, MIT licence)",
          "title": "robots.json, the cohere-ai entry's “function” field",
          "quote": "Retrieves data to provide responses to user-initiated prompts.",
          "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json",
          "checked": "2026-09-26"
        }
      ]
    },
    {
      "token": "anthropic-ai",
      "slug": "anthropic-ai",
      "operator": "Anthropic (attributed by third parties; Anthropic's crawler page does not name this token)",
      "why": "Anthropic's crawler page lists three bots and not anthropic-ai; the one it documents for training is ClaudeBot, which is in the documented group above. A third-party list still describes anthropic-ai as a training scraper.",
      "operatorSays": {
        "documentsPurpose": false,
        "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
        "checked": "2026-09-26",
        "searched": [
          { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "checked": "2026-09-26", "result": "Loaded; lists ClaudeBot, Claude-User and Claude-SearchBot. No mention of anthropic-ai." }
        ],
        "quotes": [
          {
            "quote": "Below is information on the three robots that Anthropic uses and how to set your site preferences to enable those you want to access your content and limit those you don’t.",
            "context": "The three are ClaudeBot, Claude-User and Claude-SearchBot; anthropic-ai is not among them.",
            "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
            "checked": "2026-09-26"
          }
        ]
      },
      "thirdParty": [
        {
          "who": "ai.robots.txt (community-maintained list of AI crawlers, MIT licence)",
          "title": "robots.json, the anthropic-ai entry's “description” field",
          "quote": "Scrapes data to train LLMs and AI products offered by Anthropic.",
          "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json",
          "checked": "2026-09-26"
        }
      ]
    },
    {
      "token": "FacebookBot",
      "slug": "facebookbot",
      "operator": "Meta (attributed by third parties; Meta's crawler page does not name this token)",
      "why": "Meta's crawler page lists five crawlers and not FacebookBot; the one it documents for AI training is Meta-ExternalAgent, which is in the documented group above. A third-party list says Meta once described FacebookBot as training speech recognition.",
      "operatorSays": {
        "documentsPurpose": false,
        "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
        "checked": "2026-09-24",
        "searched": [
          { "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "checked": "2026-09-24", "result": "Loaded; lists FacebookExternalHit, Meta-WebIndexer, Meta-ExternalAds, Meta-ExternalAgent and Meta-ExternalFetcher. No mention of FacebookBot. (On 2026-09-26 the page returned no readable content to a plain GET, so the 24 Sep reading stands.)" }
        ],
        "quotes": []
      },
      "thirdParty": [
        {
          "who": "ai.robots.txt (community-maintained list of AI crawlers, MIT licence)",
          "title": "robots.json, the FacebookBot entry's “description” field",
          "quote": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically.",
          "url": "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.json",
          "checked": "2026-09-26"
        }
      ]
    },
    {
      "token": "ImagesiftBot",
      "slug": "imagesiftbot",
      "operator": "Hive (ImageSift)",
      "why": "Hive documents ImagesiftBot as collecting images, page text and alt text for an image index its “web intelligence products” search. It does not call this AI training, and no third party we checked says it is.",
      "operatorSays": {
        "documentsPurpose": true,
        "quotes": [
          {
            "quote": "ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support our suite of web intelligence products",
            "context": "ImageSift Bot page, first sentence.",
            "url": "https://imagesift.com/about",
            "checked": "2026-09-26"
          },
          {
            "quote": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images.",
            "context": "FAQ, “How do we use this information?”",
            "url": "https://imagesift.com/about",
            "checked": "2026-09-26"
          }
        ]
      },
      "robotsNote": {
        "quote": "If there is no rule targeting ImagesiftBot, but there is a rule targeting Googlebot, then ImagesiftBot will follow the Googlebot directives.",
        "url": "https://imagesift.com/about",
        "checked": "2026-09-26",
        "note": "The verdict here uses RFC 9309 (its own group, else User-agent: *). If your robots.txt has no ImagesiftBot group but has a Googlebot group, Hive says ImagesiftBot follows the Googlebot group instead."
      },
      "thirdParty": [
        {
          "who": "Cloudflare",
          "title": "Declare your AIndependence: block AI bots, scrapers and crawlers with a single click (table of AI bots by share of Cloudflare-protected websites accessed)",
          "published": "2024-07-03",
          "quote": "ImagesiftBot 8.75%",
          "url": "https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click/",
          "checked": "2026-09-26"
        }
      ]
    }
  ]
}
