{
  "dataset": "AI crawler rules in robots.txt, 50 domains",
  "article": "https://proxy.wiki/research/ai-crawler-robots-policies/",
  "collected": "2026-09-01",
  "collected_utc": "2026-09-01T20:24Z",
  "method": "GET https://<host>/robots.txt with redirects followed, User-Agent 'proxy.wiki-research/1.0 (+https://proxy.wiki/research/)'. Files were split into user-agent groups per RFC 9309 (consecutive User-agent lines share the rules that follow). Verdict per agent: blocked = a group naming it has Disallow: / ; allowed = named and not disallowed from everything (or an explicit Allow: / overrides); partial = some paths disallowed; unmentioned = never named.",
  "sample": "50 domains requested, 46 returned a readable file, 18 AI user agents evaluated per site.",
  "known_gaps": [
    "Per-site verdicts were kept for 7 of the 18 agents (the columns printed in the article). For the other 11 agents only the per-agent totals and each site's count of blocks across all 18 survive; those cells are null for the 31 sites that name an agent.",
    "Sites that name no AI agent have every agent cell set to unmentioned, which follows from the definition.",
    "The wildcard verdict per site was recorded only for the four sites spot-checked in the session; the other wildcard cells are null. The aggregate is in wildcard_totals.",
    "For the four failed fetches the HTTP status code was not recorded, only the exception class."
  ],
  "agent_totals_columns": [
    "agent",
    "blocked",
    "allowed",
    "partial",
    "unmentioned"
  ],
  "agent_totals": [
    {
      "agent": "ClaudeBot",
      "blocked": 22,
      "allowed": 1,
      "partial": 1,
      "unmentioned": 22
    },
    {
      "agent": "CCBot",
      "blocked": 21,
      "allowed": 2,
      "partial": 1,
      "unmentioned": 22
    },
    {
      "agent": "Applebot-Extended",
      "blocked": 19,
      "allowed": 1,
      "partial": 1,
      "unmentioned": 25
    },
    {
      "agent": "Bytespider",
      "blocked": 18,
      "allowed": 1,
      "partial": 1,
      "unmentioned": 26
    },
    {
      "agent": "meta-externalagent",
      "blocked": 18,
      "allowed": 0,
      "partial": 1,
      "unmentioned": 27
    },
    {
      "agent": "Google-Extended",
      "blocked": 17,
      "allowed": 2,
      "partial": 1,
      "unmentioned": 26
    },
    {
      "agent": "GPTBot",
      "blocked": 16,
      "allowed": 3,
      "partial": 3,
      "unmentioned": 24
    },
    {
      "agent": "PerplexityBot",
      "blocked": 16,
      "allowed": 2,
      "partial": 2,
      "unmentioned": 26
    },
    {
      "agent": "Amazonbot",
      "blocked": 16,
      "allowed": 1,
      "partial": 3,
      "unmentioned": 26
    },
    {
      "agent": "anthropic-ai",
      "blocked": 15,
      "allowed": 2,
      "partial": 1,
      "unmentioned": 28
    },
    {
      "agent": "cohere-ai",
      "blocked": 14,
      "allowed": 1,
      "partial": 0,
      "unmentioned": 31
    },
    {
      "agent": "Diffbot",
      "blocked": 13,
      "allowed": 0,
      "partial": 1,
      "unmentioned": 32
    },
    {
      "agent": "omgili",
      "blocked": 13,
      "allowed": 0,
      "partial": 0,
      "unmentioned": 33
    },
    {
      "agent": "Claude-User",
      "blocked": 12,
      "allowed": 0,
      "partial": 2,
      "unmentioned": 32
    },
    {
      "agent": "ChatGPT-User",
      "blocked": 11,
      "allowed": 1,
      "partial": 6,
      "unmentioned": 28
    },
    {
      "agent": "Timpibot",
      "blocked": 9,
      "allowed": 0,
      "partial": 1,
      "unmentioned": 36
    },
    {
      "agent": "ImagesiftBot",
      "blocked": 8,
      "allowed": 0,
      "partial": 0,
      "unmentioned": 38
    },
    {
      "agent": "OAI-SearchBot",
      "blocked": 7,
      "allowed": 1,
      "partial": 6,
      "unmentioned": 32
    }
  ],
  "wildcard_totals": {
    "partial": 37,
    "blocked": 6,
    "unmentioned": 2,
    "allowed": 1
  },
  "summary": {
    "readable": 46,
    "name_at_least_one_ai_agent": 31,
    "name_none": 15,
    "block_gptbot_and_claudebot": 16,
    "block_claudebot_not_gptbot": 6,
    "block_gptbot_not_claudebot": 0,
    "block_gptbot_claudebot_ccbot": 15,
    "block_claudebot_not_gptbot_sites": [
      {
        "site": "arstechnica.com",
        "GPTBot": "unmentioned"
      },
      {
        "site": "investopedia.com",
        "GPTBot": "partial"
      },
      {
        "site": "theguardian.com",
        "GPTBot": "unmentioned"
      },
      {
        "site": "theverge.com",
        "GPTBot": "allowed"
      },
      {
        "site": "washingtonpost.com",
        "GPTBot": "unmentioned"
      },
      {
        "site": "wired.com",
        "GPTBot": "unmentioned"
      }
    ]
  },
  "licence": "CC BY 4.0",
  "licence_url": "https://creativecommons.org/licenses/by/4.0/",
  "attribution": "proxy.wiki",
  "null_means": "null (an empty cell in the CSV) means the verdict was not kept. 'unmentioned' is a verdict: the file never names that agent.",
  "columns": [
    "site",
    "robots_txt_readable",
    "fetch_error",
    "names_any_ai_agent",
    "wildcard_group",
    "GPTBot",
    "OAI-SearchBot",
    "ChatGPT-User",
    "ClaudeBot",
    "Claude-User",
    "anthropic-ai",
    "Google-Extended",
    "PerplexityBot",
    "CCBot",
    "Bytespider",
    "Amazonbot",
    "meta-externalagent",
    "Applebot-Extended",
    "cohere-ai",
    "Diffbot",
    "Timpibot",
    "omgili",
    "ImagesiftBot",
    "blocked_of_18"
  ],
  "rows": [
    {
      "site": "cnn.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 17,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "nytimes.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": "partial",
      "blocked_of_18": 17,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "sciencedirect.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 17,
      "GPTBot": "blocked",
      "OAI-SearchBot": "allowed",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "amazon.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 15,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "bbc.co.uk",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 15,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "bloomberg.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 15,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "nature.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 15,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "theverge.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 15,
      "GPTBot": "allowed",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "investopedia.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 14,
      "GPTBot": "partial",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "arstechnica.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 13,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "wired.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 12,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "healthline.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 11,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "techcrunch.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 11,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "unmentioned",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "washingtonpost.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 11,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "figma.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 10,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "yelp.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 10,
      "GPTBot": "blocked",
      "OAI-SearchBot": "blocked",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "ebay.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 9,
      "GPTBot": "blocked",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "theguardian.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 9,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "blocked",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "tripadvisor.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 9,
      "GPTBot": "blocked",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "partial",
      "CCBot": "blocked",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "medium.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 6,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "blocked",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "jstor.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 4,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "unmentioned",
      "CCBot": "blocked",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "webmd.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 3,
      "GPTBot": "blocked",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "blocked",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "blocked",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "zillow.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 3,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "indeed.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 1,
      "GPTBot": "partial",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "partial",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "partial",
      "PerplexityBot": "partial",
      "CCBot": "partial",
      "Bytespider": "partial",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "nerdwallet.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 1,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "blocked",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "notion.so",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 1,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "reuters.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 1,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "booking.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "britannica.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "cloudflare.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "allowed",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "allowed",
      "PerplexityBot": "allowed",
      "CCBot": "allowed",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "en.wikipedia.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": "partial",
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "etsy.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "forbes.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "github.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": "partial",
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "imdb.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "linkedin.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "mozilla.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "nodejs.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "python.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "reddit.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": "blocked",
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "shopify.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "substack.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "vercel.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "wordpress.org",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "allowed",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": null,
      "ClaudeBot": "allowed",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "allowed",
      "PerplexityBot": "allowed",
      "CCBot": "allowed",
      "Bytespider": "allowed",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "wsj.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": true,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "partial",
      "OAI-SearchBot": "partial",
      "ChatGPT-User": null,
      "ClaudeBot": "unmentioned",
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "ycombinator.com",
      "robots_txt_readable": true,
      "fetch_error": null,
      "names_any_ai_agent": false,
      "wildcard_group": null,
      "blocked_of_18": 0,
      "GPTBot": "unmentioned",
      "OAI-SearchBot": "unmentioned",
      "ChatGPT-User": "unmentioned",
      "ClaudeBot": "unmentioned",
      "Claude-User": "unmentioned",
      "anthropic-ai": "unmentioned",
      "Google-Extended": "unmentioned",
      "PerplexityBot": "unmentioned",
      "CCBot": "unmentioned",
      "Bytespider": "unmentioned",
      "Amazonbot": "unmentioned",
      "meta-externalagent": "unmentioned",
      "Applebot-Extended": "unmentioned",
      "cohere-ai": "unmentioned",
      "Diffbot": "unmentioned",
      "Timpibot": "unmentioned",
      "omgili": "unmentioned",
      "ImagesiftBot": "unmentioned"
    },
    {
      "site": "mayoclinic.org",
      "robots_txt_readable": false,
      "fetch_error": "HTTPError",
      "names_any_ai_agent": null,
      "wildcard_group": null,
      "blocked_of_18": null,
      "GPTBot": null,
      "OAI-SearchBot": null,
      "ChatGPT-User": null,
      "ClaudeBot": null,
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": null,
      "PerplexityBot": null,
      "CCBot": null,
      "Bytespider": null,
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "pinterest.com",
      "robots_txt_readable": false,
      "fetch_error": "HTTPError",
      "names_any_ai_agent": null,
      "wildcard_group": null,
      "blocked_of_18": null,
      "GPTBot": null,
      "OAI-SearchBot": null,
      "ChatGPT-User": null,
      "ClaudeBot": null,
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": null,
      "PerplexityBot": null,
      "CCBot": null,
      "Bytespider": null,
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "quora.com",
      "robots_txt_readable": false,
      "fetch_error": "HTTPError",
      "names_any_ai_agent": null,
      "wildcard_group": null,
      "blocked_of_18": null,
      "GPTBot": null,
      "OAI-SearchBot": null,
      "ChatGPT-User": null,
      "ClaudeBot": null,
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": null,
      "PerplexityBot": null,
      "CCBot": null,
      "Bytespider": null,
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    },
    {
      "site": "stackoverflow.com",
      "robots_txt_readable": false,
      "fetch_error": "HTTPError",
      "names_any_ai_agent": null,
      "wildcard_group": null,
      "blocked_of_18": null,
      "GPTBot": null,
      "OAI-SearchBot": null,
      "ChatGPT-User": null,
      "ClaudeBot": null,
      "Claude-User": null,
      "anthropic-ai": null,
      "Google-Extended": null,
      "PerplexityBot": null,
      "CCBot": null,
      "Bytespider": null,
      "Amazonbot": null,
      "meta-externalagent": null,
      "Applebot-Extended": null,
      "cohere-ai": null,
      "Diffbot": null,
      "Timpibot": null,
      "omgili": null,
      "ImagesiftBot": null
    }
  ]
}
