{
  "slug": "real-noise-verdict",
  "name": "Real, noise or can't-tell",
  "aliases": [
    "Verdict",
    "Significance verdict"
  ],
  "definition": "A three-way ruling on a period-over-period change: real, noise, or can't-tell.",
  "category": "Statistics",
  "type": "statistical verdict",
  "unit": "one of real, noise, can't-tell",
  "direction": "neutral",
  "verification": "verified",
  "calculation": "Derived, never eyeballed. A p-value below the threshold gives real. Otherwise the sample is checked for adequate power: a null result with adequate power gives noise, and a null result without it gives can't-tell. Power is a floor, not a computed power figure, at least eight time buckets in each period, the same floor for every metric including conversion rate. One further route to can't-tell bypasses the p-value entirely: when the change in the displayed period aggregate and the change in the per-bucket means point in opposite directions, the test result is not transferred to the displayed number at all.",
  "aggregation": "One verdict per test.",
  "grain": "One verdict per test.",
  "dimensions": [],
  "requiredFilters": [
    "date_range",
    "comparison_range"
  ],
  "sources": [
    "web-analytics"
  ],
  "reports": [
    "ai-overview-report",
    "ai-referral-traffic-report",
    "ai-visibility-report",
    "cannibalization-report",
    "competitor-search-report",
    "content-decay-report",
    "content-performance-report",
    "crawl-log-report",
    "ctr-opportunity-report",
    "keyword-ranking-report",
    "monthly-executive-search-report",
    "paid-organic-overlap-report",
    "paid-search-performance-report",
    "search-market-share-report",
    "search-to-revenue-report",
    "serp-volatility-report",
    "weekly-search-report"
  ],
  "skills": [
    "significance-referee",
    "anomaly-investigator",
    "monthly-exec-review"
  ],
  "verbs": [
    "pop_significance"
  ],
  "rungs": [
    "R5"
  ],
  "levers": [],
  "questions": [
    "Is this drop real or just noise?",
    "Can we call this month a real improvement?"
  ],
  "interpretation": "The three-word vocabulary exists because two words are not enough: an underpowered null is not stability. Can't-tell means the test lacked the sample to find a change of the size that would matter, and it is the correct answer more often than teams expect on low-traffic segments. The vocabulary is fixed, probably real, trending significant and basically stable are not options.",
  "caveats": [
    "Power here is a fixed floor on the number of time buckets, not a computed statistical power figure. A change smaller than the floor could detect will still be called noise once the floor is cleared.",
    "The floor is eight buckets per period, and it is the same for every metric, there is no separate sample-size floor for rates, because rates are tested the same way as counts.",
    "Because the floor counts buckets, the bucket size chosen for the test decides whether a period is deemed powered at all. A month tested weekly cannot clear eight buckets; the same month tested daily clears it easily on the same data.",
    "Only the web-analytics period-over-period tool emits this verdict. The cross-domain tool returns a two-state flag, so a can't-tell there is reported as not significant.",
    "A real verdict is still not a cause; ruling on existence is the whole of the opinion.",
    "Too few buckets, or no variation within either period, produces a can't-tell with an explanatory note rather than a test result, and the change itself is still reported beside it."
  ],
  "freshness": "loads nightly, a same-day question gets yesterday's number, labelled as yesterday's.",
  "failureModes": [
    "Reporting a can't-tell as stable or unchanged.",
    "Reading the power floor as a computed power figure, or as a floor on traffic rather than on buckets.",
    "Comparing verdicts produced at different bucket sizes as though they were equally powered.",
    "Expecting a verdict from the cross-domain tool, which does not produce one."
  ],
  "notSameAs": [
    {
      "slug": "significant-flag",
      "why": "The flag has two states and collapses an underpowered null into the same answer as a well-powered one. The verdict keeps them apart, which is the entire reason it exists."
    },
    {
      "slug": "practical-effect-magnitude",
      "why": "Real means distinguishable from noise, not large enough to act on. No effect size is computed anywhere, so the verdict says nothing about magnitude."
    },
    {
      "slug": "pop-delta",
      "why": "A delta states how far the number moved; a significance verdict states whether the move is distinguishable from noise at an alpha. The verdict is a p-value threshold only, no effect size is computed anywhere, so a significant change can be trivially small and a large delta can fail the test.",
      "mirroredFrom": "pop-delta"
    },
    {
      "slug": "pop-delta-pct",
      "why": "A relative change is a magnitude; a significance verdict is a test result at an alpha. Neither implies the other, and no effect size is computed on either side.",
      "mirroredFrom": "pop-delta-pct"
    },
    {
      "slug": "ranking-movement",
      "why": "Movement classification is a fixed ±10% cut on the relative change; a significance verdict is a two-sample test at an alpha. Being a gainer says nothing about whether the change is distinguishable from noise, and no effect size is computed on either side.",
      "mirroredFrom": "ranking-movement"
    }
  ],
  "related": [
    "significant-flag",
    "significance-p-value",
    "conversion-rate-change"
  ],
  "workflows": [],
  "lastVerified": "2026-08-04"
}