{
  "slug": "significant-flag",
  "name": "Significant",
  "aliases": [
    "Statistically significant",
    "Significance flag"
  ],
  "definition": "A true or false flag set when the p-value falls below the chosen threshold.",
  "category": "Statistics",
  "type": "statistical verdict",
  "unit": "true or false, or absent",
  "direction": "neutral",
  "verification": "verified",
  "calculation": "The p-value compared against the threshold, which defaults to 0.05 and is a caller-settable parameter. That comparison is the whole of the flag, nothing else feeds it. The one exception is a refusal rather than a computation: where the displayed period change and the per-bucket change point in opposite directions, the period-over-period tool emits no flag at all rather than attaching the bucket-level result to a number the test did not measure.",
  "aggregation": "One flag per test.",
  "grain": "One flag per test.",
  "dimensions": [],
  "requiredFilters": [
    "date_range",
    "comparison_range"
  ],
  "sources": [
    "search-console",
    "web-analytics",
    "google-ads",
    "rank-tracking",
    "lighthouse"
  ],
  "reports": [
    "keyword-ranking-report"
  ],
  "skills": [
    "significance-referee",
    "anomaly-investigator",
    "monthly-exec-review",
    "rank-movement-review"
  ],
  "verbs": [
    "pop_significance",
    "significance_check"
  ],
  "rungs": [
    "R5",
    "R2"
  ],
  "levers": [],
  "questions": [
    "Is that significant?",
    "Did it really change?"
  ],
  "interpretation": "A threshold crossing, not a judgment about importance. The threshold is a parameter, so a flag is only comparable against another flag computed at the same threshold. Where a decision is at stake, the verdict field is the better read, because it separates a well-powered null result from an underpowered one.",
  "caveats": [
    "The flag is the p-value below the threshold and nothing else. It carries no effect size, no minimum detectable effect and no confidence interval, none of those is computed anywhere in the tool surface.",
    "No multiple-comparison correction is applied, so a batch of tests will produce false flags at roughly the threshold rate.",
    "A false flag does not mean no change. The cross-domain tool reports only this flag, so an underpowered null there is indistinguishable from a well-powered one, the rate-aware tool's verdict field exists to make that distinction.",
    "The threshold is caller-settable, so two flags are only comparable when both used the same one.",
    "An absent flag is a third state, not a false one. It means the tool declined to attach a test result to the displayed change, and reading it as not-significant inverts the refusal into a finding.",
    "Significance is not causation."
  ],
  "freshness": "varies by site, priority URLs can run nightly; most pages far less often. The collection date rides on the card rather than being assumed.",
  "failureModes": [
    "Reading a false flag as evidence of stability.",
    "Treating an absent flag as a false one.",
    "Comparing flags computed at different thresholds.",
    "Reporting the significant rows from a large batch without noting that nothing corrected for the batch size."
  ],
  "notSameAs": [
    {
      "slug": "practical-effect-magnitude",
      "why": "This flag is a threshold crossing on a p-value only. No effect size is ever computed by any tool, so the flag cannot speak to whether a change is large enough to act on."
    },
    {
      "slug": "real-noise-verdict",
      "why": "The flag has two states; the verdict has three, because it separates a null result with adequate power from a null result without it. A false flag can mean either."
    }
  ],
  "related": [
    "significance-p-value",
    "real-noise-verdict",
    "practical-effect-magnitude"
  ],
  "workflows": [],
  "lastVerified": "2026-08-04"
}