{
  "name": "resulthack AI-training opt-out signals",
  "description": "The machine-readable signals a site can send to say it does not want its content used for AI training, who defined each one, where its specification is, and which AI crawler operators document honouring which signal. Every operator cell is taken from that operator's own documentation, with its URL and the date it was read. \"not documented\" means the page named in that cell was read on that date and does not mention the signal; it does not mean the operator ignores it.",
  "homepage": "https://resulthack.com/opt-out/",
  "updated": "2026-09-25",
  "statuses": {
    "documented": "The operator's own documentation says it reads this signal (quoted).",
    "declined": "The operator's own documentation says it prefers another signal over this one (quoted).",
    "not documented": "The operator page named in the cell does not mention this signal. Absence of a statement, not a statement of non-compliance."
  },
  "law": {
    "title": "Directive (EU) 2019/790 on copyright and related rights in the Digital Single Market, Article 4(3)",
    "quote": "The exception or limitation provided for in paragraph 1 shall apply on condition that the use of works and other subject matter referred to in that paragraph has not been expressly reserved by their rightholders in an appropriate manner, such as machine-readable means in the case of content made publicly available online.",
    "url": "https://eur-lex.europa.eu/eli/dir/2019/790/oj",
    "read": "2026-09-25",
    "readNote": "EUR-Lex answered our plain GET with a browser challenge (HTTP 202, no text), so the quote was read from the Publications Office copy of the same official text, given under source.",
    "source": "https://publications.europa.eu/resource/celex/32019L0790"
  },
  "signals": [
    {
      "id": "robots-token",
      "name": "Per-agent robots.txt tokens",
      "where": "robots.txt at the site root",
      "definedBy": "robots.txt is RFC 9309 (IETF, 2022). Each training token is defined by the operator that reads it: GPTBot by OpenAI, Google-Extended by Google, and so on.",
      "specs": [
        {
          "title": "RFC 9309: Robots Exclusion Protocol",
          "url": "https://www.rfc-editor.org/rfc/rfc9309",
          "read": "2026-09-25"
        }
      ]
    },
    {
      "id": "content-signal",
      "name": "Content Signals (Content-Signal line in robots.txt)",
      "where": "robots.txt, inside a User-agent group",
      "definedBy": "Cloudflare, the Content Signals Policy, announced 24 Sep 2025. It defines search, ai-input and ai-train (yes or no); Cloudflare's robots.txt documentation (last updated 3 Aug 2026) adds a fourth field, use, with the values immediate, reference or full.",
      "specs": [
        {
          "title": "Content Signals (contentsignals.org, the policy text and examples)",
          "url": "https://contentsignals.org/",
          "read": "2026-09-25"
        },
        {
          "title": "Cloudflare blog: Giving users choice with Cloudflare's new Content Signals Policy (24 Sep 2025)",
          "url": "https://blog.cloudflare.com/content-signals-policy/",
          "read": "2026-09-25"
        },
        {
          "title": "Cloudflare docs: robots.txt setting (use= field; last updated 3 Aug 2026)",
          "url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/",
          "read": "2026-09-25"
        }
      ]
    },
    {
      "id": "tdmrep",
      "name": "TDMRep (TDM Reservation Protocol)",
      "where": "/.well-known/tdmrep.json, the tdm-reservation and tdm-policy HTTP headers, and meta tags of the same names",
      "definedBy": "The W3C Text and Data Mining Reservation Protocol Community Group; Final Community Group Report of 10 May 2024, editor Laurent Le Meur (EDRLab). The report itself says it is not a W3C Standard nor on the W3C Standards Track.",
      "specs": [
        {
          "title": "TDM Reservation Protocol (TDMRep), Final Community Group Report, 10 May 2024",
          "url": "https://www.w3.org/community/reports/tdmrep/CG-FINAL-tdmrep-20240510/",
          "read": "2026-09-25"
        }
      ]
    },
    {
      "id": "noai",
      "name": "noai and noimageai",
      "where": "a robots meta tag, or an X-Robots-Tag response header",
      "definedBy": "DeviantArt, November 2022, for its own artists' pages; other sites copied it. It is not in Google's or Bing's robots meta documentation and has no published specification.",
      "specs": [
        {
          "title": "DeviantArt: UPDATE All Deviations Are Opted Out of AI Datasets (Nov 2022)",
          "url": "https://www.deviantart.com/team/journal/UPDATE-All-Deviations-Are-Opted-Out-of-AI-Datasets-934500371",
          "read": "2026-09-25",
          "readNote": "The page answered our fetch with HTTP 403; the origin and date are as the page is indexed by search engines and reported by the press at the time."
        }
      ]
    },
    {
      "id": "ai-txt",
      "name": "ai.txt (Spawning)",
      "where": "/ai.txt at the site root",
      "definedBy": "Spawning, the company behind Have I Been Trained and the Do Not Train registry. It uses robots.txt syntax (User-Agent, Allow, Disallow with file-type patterns). There is no published specification beyond Spawning's generator and FAQ; Spawning says its API passes ai.txt permissions to \"a growing list of AI researchers and partners\" without naming them.",
      "specs": [
        {
          "title": "Spawning: ai.txt generator and FAQ",
          "url": "https://site.spawning.ai/spawning-ai-txt",
          "read": "2026-09-25"
        }
      ]
    }
  ],
  "operators": [
    {
      "operator": "OpenAI",
      "token": "GPTBot",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "GPTBot",
          "quote": "Disallowing GPTBot indicates a site’s content should not be used in training generative AI foundation models.",
          "url": "https://developers.openai.com/api/docs/bots",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://developers.openai.com/api/docs/bots",
          "read": "2026-09-25",
          "note": "Not mentioned on OpenAI’s crawler page (Overview of OpenAI Crawlers), read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://developers.openai.com/api/docs/bots",
          "read": "2026-09-25",
          "note": "Not mentioned on OpenAI’s crawler page (Overview of OpenAI Crawlers), read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://developers.openai.com/api/docs/bots",
          "read": "2026-09-25",
          "note": "Not mentioned on OpenAI’s crawler page (Overview of OpenAI Crawlers), read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://developers.openai.com/api/docs/bots",
          "read": "2026-09-25",
          "note": "Not mentioned on OpenAI’s crawler page (Overview of OpenAI Crawlers), read 2026-09-25."
        }
      }
    },
    {
      "operator": "Anthropic",
      "token": "ClaudeBot",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "ClaudeBot",
          "quote": "When a site restricts ClaudeBot access, it signals that the site's future materials should be excluded from our AI model training datasets.",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "read": "2026-09-25",
          "note": "Not mentioned on Anthropic’s help-centre article on its crawlers, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "read": "2026-09-25",
          "note": "Not mentioned on Anthropic’s help-centre article on its crawlers, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "read": "2026-09-25",
          "note": "Not mentioned on Anthropic’s help-centre article on its crawlers, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "read": "2026-09-25",
          "note": "Not mentioned on Anthropic’s help-centre article on its crawlers, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Google",
      "token": "Google-Extended",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "Google-Extended",
          "quote": "Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models that power Gemini Apps and Vertex AI API for Gemini and for grounding (providing content from the Google Search index to the model at prompt time to improve factuality and relevancy) in Gemini Apps and Grounding with Google Search on Vertex AI.",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
          "read": "2026-09-25",
          "note": "Google-Extended also controls grounding in Gemini Apps, not only training. Google says it does not affect inclusion in Google Search."
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Google’s list of common crawlers, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Google’s list of common crawlers, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Google’s list of common crawlers, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Google’s list of common crawlers, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Apple",
      "token": "Applebot-Extended",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "Applebot-Extended",
          "quote": "With Applebot-Extended, web publishers can choose to opt out of their website content being used to train Apple’s general purpose foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools.",
          "url": "https://support.apple.com/en-us/119829",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://support.apple.com/en-us/119829",
          "read": "2026-09-25",
          "note": "Not mentioned on Apple’s About Applebot page, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://support.apple.com/en-us/119829",
          "read": "2026-09-25",
          "note": "Not mentioned on Apple’s About Applebot page, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://support.apple.com/en-us/119829",
          "read": "2026-09-25",
          "note": "Not mentioned on Apple’s About Applebot page, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://support.apple.com/en-us/119829",
          "read": "2026-09-25",
          "note": "Not mentioned on Apple’s About Applebot page, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Meta",
      "token": "Meta-ExternalAgent",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "Meta-ExternalAgent",
          "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Meta’s Web Crawlers page, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Meta’s Web Crawlers page, read 2026-09-25."
        },
        "noai": {
          "status": "declined",
          "quote": "We make it easy for site managers and content owners to indicate their preferences by using industry-standard practices like robots.txt rather than non-standard formats like NoAI tags.",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
          "read": "2026-09-25"
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
          "read": "2026-09-25",
          "note": "Not mentioned on Meta’s Web Crawlers page, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Amazon",
      "token": "Amazonbot",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "Amazonbot",
          "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol , honoring the user-agent and the allow/disallow directives.",
          "url": "https://developer.amazon.com/amazonbot",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://developer.amazon.com/amazonbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Amazon’s About Amazonbot page, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://developer.amazon.com/amazonbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Amazon’s About Amazonbot page, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://developer.amazon.com/amazonbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Amazon’s About Amazonbot page, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://developer.amazon.com/amazonbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Amazon’s About Amazonbot page, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Mistral AI",
      "token": "MistralAI-Training",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "MistralAI-Training",
          "quote": "MistralAI-Training crawls web content to help build datasets for training Mistral generative AI models. Webmasters can disallow this user agent in their robots.txt file.",
          "url": "https://docs.mistral.ai/robots",
          "read": "2026-09-25"
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://docs.mistral.ai/robots",
          "read": "2026-09-25",
          "note": "Not mentioned on Mistral’s crawler page, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://docs.mistral.ai/robots",
          "read": "2026-09-25",
          "note": "Not mentioned on Mistral’s crawler page, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://docs.mistral.ai/robots",
          "read": "2026-09-25",
          "note": "Not mentioned on Mistral’s crawler page, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://docs.mistral.ai/robots",
          "read": "2026-09-25",
          "note": "Not mentioned on Mistral’s crawler page, read 2026-09-25."
        }
      }
    },
    {
      "operator": "Common Crawl",
      "token": "CCBot",
      "claims": {
        "robots-token": {
          "status": "documented",
          "token": "CCBot",
          "quote": "Add these lines to your robots.txt file and our crawler will stop crawling your website:",
          "url": "https://commoncrawl.org/ccbot",
          "read": "2026-09-25",
          "note": "This stops CCBot crawling at all. Common Crawl’s own pages do not describe CCBot as an AI-training crawler; its dataset is widely used for training by others."
        },
        "content-signal": {
          "status": "not documented",
          "url": "https://commoncrawl.org/ccbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Common Crawl’s CCBot page, read 2026-09-25."
        },
        "tdmrep": {
          "status": "not documented",
          "url": "https://commoncrawl.org/ccbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Common Crawl’s CCBot page, read 2026-09-25."
        },
        "noai": {
          "status": "not documented",
          "url": "https://commoncrawl.org/ccbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Common Crawl’s CCBot page, read 2026-09-25."
        },
        "ai-txt": {
          "status": "not documented",
          "url": "https://commoncrawl.org/ccbot",
          "read": "2026-09-25",
          "note": "Not mentioned on Common Crawl’s CCBot page, read 2026-09-25."
        }
      }
    }
  ]
}