{
  "slug": "unique-pages-crawled-by-bots",
  "name": "Unique pages crawled",
  "aliases": [
    "Unique URLs fetched by bots",
    "Pages reached by bots"
  ],
  "definition": "The number of distinct URLs a named crawler fetched in the period, counted from server logs.",
  "category": "Crawl and logs",
  "type": "raw measure",
  "unit": "distinct URLs",
  "direction": "neutral",
  "verification": "verified",
  "calculation": "A distinct count of the language-variant-normalised full URL over the matching log records.",
  "aggregation": "A distinct count, so it is never summed across bots or buckets, the same page fetched by two bots is one page for each of them and cannot be added.",
  "grain": "One count per bot per date range, or per bot per time bucket.",
  "dimensions": [
    "bot_type",
    "date"
  ],
  "requiredFilters": [
    "date_range"
  ],
  "sources": [
    "server-logs"
  ],
  "reports": [
    "ai-bot-activity-report",
    "crawl-log-report",
    "internal-linking-report",
    "technical-seo-audit-report"
  ],
  "skills": [
    "ai-crawler-readiness",
    "crawl-efficiency-review",
    "technical-health-audit"
  ],
  "verbs": [
    "ai_crawler_analysis"
  ],
  "rungs": [
    "R1"
  ],
  "levers": [
    "L1"
  ],
  "questions": [
    "How much of our site are AI crawlers reaching?",
    "Is crawl coverage growing?"
  ],
  "interpretation": "The coverage half of crawl activity. Read against the hit count it separates two different situations that look alike in a total: broad shallow crawling, and heavy repeat fetching of a small set.",
  "caveats": [
    "A distinct count, so bot-level or bucket-level values cannot be added into a site total.",
    "URLs are normalised across language variants before counting, so localised versions of one page collapse together.",
    "It counts what was fetched, not what exists, it is not a share of your site unless you separately know the site's page count.",
    "Same bot-identification and CDN caveats as the hit count: user-agent matching only, and origin-served requests only."
  ],
  "freshness": "hours to nightly, depends on how logs are shipped; the card carries the ingestion time.",
  "failureModes": [
    "Summing per-bot unique-page counts.",
    "Presenting it as a percentage of the site without a denominator from the crawl."
  ],
  "notSameAs": [
    {
      "slug": "bot-hits",
      "why": "One counts requests, the other counts distinct URLs. The ratio between them is the crawl-repetition signal and neither number carries it alone."
    }
  ],
  "related": [
    "bot-hits",
    "crawler-readiness",
    "crawl-depth"
  ],
  "workflows": [],
  "lastVerified": "2026-08-04"
}