{
  "finished": true,
  "generated_at": "2026-09-06T23:21:12.818Z",
  "scope": "Retry unreadable robots.txt for original observed paths; not a new HTTP access test.",
  "target_domains": 1746,
  "target_documents": 2275,
  "attempts": 6307,
  "documents_resolved": 269,
  "fully_resolved_search_permissions": 257,
  "partly_resolved_search_permissions": 0,
  "before": {
    "allowed": 3300,
    "restriction": 338,
    "unknown": 1743
  },
  "after": {
    "allowed": 3539,
    "restriction": 356,
    "unknown": 1486
  },
  "changed_domains": 257,
  "bot_counts": [
    {
      "key": "oai-search",
      "label": "OAI-SearchBot",
      "allowed": 3705,
      "disallowed": 187,
      "unknown": 1489
    },
    {
      "key": "chatgpt-user",
      "label": "ChatGPT-User",
      "allowed": 3608,
      "disallowed": 285,
      "unknown": 1488
    },
    {
      "key": "gptbot",
      "label": "GPTBot",
      "allowed": 3307,
      "disallowed": 594,
      "unknown": 1480
    },
    {
      "key": "claude-search",
      "label": "Claude-SearchBot",
      "allowed": 3690,
      "disallowed": 203,
      "unknown": 1488
    },
    {
      "key": "claude-user",
      "label": "Claude-User",
      "allowed": 3706,
      "disallowed": 188,
      "unknown": 1487
    },
    {
      "key": "claudebot",
      "label": "ClaudeBot",
      "allowed": 3328,
      "disallowed": 571,
      "unknown": 1482
    },
    {
      "key": "perplexity",
      "label": "PerplexityBot",
      "allowed": 3557,
      "disallowed": 338,
      "unknown": 1486
    },
    {
      "key": "perplexity-u",
      "label": "Perplexity-User",
      "allowed": 3700,
      "disallowed": 195,
      "unknown": 1486
    },
    {
      "key": "googlebot",
      "label": "Googlebot",
      "allowed": 3865,
      "disallowed": 26,
      "unknown": 1490
    },
    {
      "key": "google-ext",
      "label": "Google-Extended",
      "allowed": 3411,
      "disallowed": 486,
      "unknown": 1484
    },
    {
      "key": "bingbot",
      "label": "Bingbot",
      "allowed": 3839,
      "disallowed": 54,
      "unknown": 1488
    },
    {
      "key": "meta",
      "label": "Meta-ExternalAgent",
      "allowed": 3382,
      "disallowed": 512,
      "unknown": 1487
    },
    {
      "key": "amazonbot",
      "label": "Amazonbot",
      "allowed": 3422,
      "disallowed": 474,
      "unknown": 1485
    },
    {
      "key": "mistral-user",
      "label": "MistralAI-User",
      "allowed": 3703,
      "disallowed": 191,
      "unknown": 1487
    },
    {
      "key": "ccbot",
      "label": "CCBot",
      "allowed": 3231,
      "disallowed": 668,
      "unknown": 1482
    },
    {
      "key": "applebot",
      "label": "Applebot",
      "allowed": 3801,
      "disallowed": 92,
      "unknown": 1488
    },
    {
      "key": "applebot-ext",
      "label": "Applebot-Extended",
      "allowed": 3425,
      "disallowed": 471,
      "unknown": 1485
    }
  ],
  "all17_disallowed": 20,
  "gpt_training_disallowed_search_allowed": 403
}