{
  "slug": "crawl-log-report",
  "name": "Crawl log report",
  "aliases": [
    "Log file report",
    "Server log analysis",
    "Crawl budget report",
    "Bot crawl report",
    "Logfile insights"
  ],
  "audience": "Technical SEO",
  "job": "Show what search crawlers actually fetched, with what status, and where crawl effort is being spent or wasted.",
  "cadence": "Weekly",
  "summary": "Server-log truth about crawling: requests by bot, status-code distribution, response times, which URL groups absorb the crawl budget and which valuable pages are fetched rarely or not at all. It is the R1 gate, nothing above it is possible for a page no crawler fetched, and it is the single most-run family in the estate, which reflects how often it is checked rather than how rarely it changes.",
  "headline": [
    "bot-hits",
    "unique-pages-crawled-by-bots",
    "status-code-hits"
  ],
  "diagnostic": [
    "indexability-flags",
    "crawl-depth",
    "incoming-internal-links",
    "pop-delta"
  ],
  "guardrail": [
    "verified-vs-spoofed-googlebot",
    "crawler-response-time",
    "real-noise-verdict"
  ],
  "sources": [
    "server-logs",
    "site-crawl"
  ],
  "skills": [
    "crawl-efficiency-review",
    "technical-health-audit",
    "anomaly-investigator"
  ],
  "rungs": [
    "R1"
  ],
  "questions": [
    "What is Googlebot crawling on our site?",
    "Where is crawl budget being wasted?",
    "Are crawlers hitting errors?",
    "Which important pages are not being crawled?"
  ],
  "workflows": [
    "subpage-portfolio-audit"
  ],
  "caveats": "Log coverage depends on which hosts, CDNs and edge layers forward logs, a bot missing from the file may be blocked at the edge rather than absent, and the two look identical here. Bot identity is claimed in the user agent unless reverse-DNS verification is in place, so 'Googlebot' counts include impostors. Log delivery lags from hours to overnight and varies by site; check the max available date before reading the last day.",
  "verification": "verified",
  "lastVerified": "2026-08-04"
}