"""Tests for the four-verdict taxonomy.

The distinction these pin down — EQUIVALENT is a finding, INCONCLUSIVE is an
admission — is the product's core UX claim. Collapsing the two is the failure
mode, so most of these tests are about keeping them apart.
"""

from __future__ import annotations

import pytest

from ablate.stats import Verdict, classify, describe
from ablate.stats.bootstrap import BootstrapResult


def _result(delta: float, lo: float | None, hi: float | None, relative: bool = False):
    return BootstrapResult(
        delta=delta,
        lo=lo,
        hi=hi,
        alpha=0.05,
        n_tasks=20,
        n_resamples=10_000,
        relative=relative,
        method="paired cluster bootstrap, BCa",
        ci_withheld=None if lo is not None else "too few tasks",
    )


def _classify(delta, lo, hi, **kw):
    return classify(_result(delta, lo, hi), metric="pass rate", margin=0.05, **kw)


# --- the four cells ---------------------------------------------------------


def test_improvement_when_interval_clears_the_margin():
    assert _classify(0.09, 0.06, 0.15).verdict is Verdict.IMPROVEMENT


def test_regression_when_interval_sits_below_the_negative_margin():
    assert _classify(-0.12, -0.19, -0.06).verdict is Verdict.REGRESSION


def test_equivalent_when_interval_fits_inside_the_margin():
    assert _classify(0.025, -0.041, 0.049).verdict is Verdict.EQUIVALENT


def test_inconclusive_when_interval_straddles_the_margin():
    assert _classify(-0.03, -0.18, 0.11).verdict is Verdict.INCONCLUSIVE


def test_equivalent_and_inconclusive_are_visually_distinct():
    """They are opposite findings and must never share a colour."""
    assert Verdict.EQUIVALENT.colour != Verdict.INCONCLUSIVE.colour
    assert Verdict.EQUIVALENT.colour != "grey"  # a finding, not an absence
    assert len({v.colour for v in Verdict}) == 4


def test_boundaries_are_inclusive_on_the_equivalent_side():
    """An interval landing exactly on ±margin is equivalent, not inconclusive."""
    assert _classify(0.0, -0.05, 0.05).verdict is Verdict.EQUIVALENT


def test_just_outside_the_margin_is_not_equivalent():
    assert _classify(0.0, -0.0501, 0.05).verdict is Verdict.INCONCLUSIVE


# --- metric direction -------------------------------------------------------


def test_lower_is_better_metrics_invert_the_verdict():
    """Fewer tokens is better, so a negative delta is an improvement for A."""
    tokens = classify(
        _result(-0.294, -0.311, -0.278, relative=True),
        metric="tokens per turn",
        margin=0.05,
        higher_is_better=False,
    )
    assert tokens.verdict is Verdict.IMPROVEMENT

    rate = _classify(-0.294, -0.311, -0.278)
    assert rate.verdict is Verdict.REGRESSION


def test_benefit_interval_is_ordered_after_flipping():
    mv = classify(
        _result(-0.30, -0.40, -0.20), metric="tokens", margin=0.05, higher_is_better=False
    )
    assert mv.benefit_lo <= mv.benefit_hi
    assert mv.benefit_lo == pytest.approx(0.20)
    assert mv.benefit_hi == pytest.approx(0.40)


# --- missing intervals ------------------------------------------------------


def test_a_missing_interval_is_inconclusive_never_equivalent():
    """A point estimate cannot establish equivalence. Treating a withheld
    interval as "no difference found" would invert the tool's purpose."""
    mv = _classify(0.001, None, None)
    assert mv.verdict is Verdict.INCONCLUSIVE
    assert not mv.has_ci


def test_missing_interval_describes_the_reason():
    assert "too few tasks" in describe(_classify(0.0, None, None))


# --- the sentences ----------------------------------------------------------


def test_no_verdict_prints_a_bare_not_significant():
    """Rule 1: never print "no significant difference" without an interval."""
    for delta, lo, hi in [(0.09, 0.06, 0.15), (-0.12, -0.19, -0.06), (0.02, -0.04, 0.05), (0.0, -0.18, 0.11)]:
        sentence = describe(_classify(delta, lo, hi))
        assert "not significant" not in sentence.lower()
        assert "%" in sentence or "pp" in sentence


def test_equivalent_reads_as_a_positive_finding():
    sentence = describe(_classify(0.025, -0.041, 0.049))
    assert "No detectable difference" in sentence
    assert "at most" in sentence
    assert "Underpowered" not in sentence


def test_inconclusive_quotes_the_price_of_an_answer():
    """Rule 3: every INCONCLUSIVE ends with what a conclusive answer costs."""
    mv = _classify(-0.03, -0.18, 0.11)
    assert mv.requirement is not None
    assert mv.requirement.n_runs > 0

    sentence = describe(mv)
    assert "Underpowered" in sentence
    assert "consistent with anything from" in sentence
    assert "runs" in sentence


def test_decisive_verdicts_do_not_quote_a_price():
    assert _classify(0.09, 0.06, 0.15).requirement is None
    assert _classify(0.025, -0.041, 0.049).requirement is None


def test_inconclusive_admits_when_no_suite_size_would_help():
    mv = classify(_result(0.0, -0.5, 0.5), metric="pass rate", margin=0.0005)
    assert mv.verdict is Verdict.INCONCLUSIVE
    assert not mv.requirement.feasible
    assert "not reachable" in describe(mv)


def test_relative_metrics_render_as_percent_not_points():
    sentence = describe(
        classify(
            _result(-0.294, -0.311, -0.278, relative=True),
            metric="tokens per turn",
            margin=0.05,
            higher_is_better=False,
        )
    )
    assert "%" in sentence
    assert "pp" not in sentence


def test_describe_uses_the_supplied_arm_labels():
    sentence = describe(_classify(0.09, 0.06, 0.15), label_a="CLAUDE.md", label_b="no config")
    assert "CLAUDE.md" in sentence


def test_only_decisive_verdicts_are_decisive():
    assert Verdict.IMPROVEMENT.is_decisive
    assert Verdict.REGRESSION.is_decisive
    assert not Verdict.EQUIVALENT.is_decisive
    assert not Verdict.INCONCLUSIVE.is_decisive


# --- the two shapes of INCONCLUSIVE -----------------------------------------


def test_an_interval_excluding_zero_is_not_called_underpowered():
    """[-57, -28] against a +/-30pp margin is inconclusive in TOST terms, but
    calling it "underpowered" beside an interval that plainly excludes zero
    reads as nonsense."""
    mv = classify(_result(-0.42, -0.57, -0.28), metric="pass rate", margin=0.30)
    assert mv.verdict is Verdict.INCONCLUSIVE
    assert mv.excludes_zero

    sentence = describe(mv)
    assert "Underpowered" not in sentence
    assert "a real effect" in sentence
    assert "--margin" in sentence


def test_an_interval_spanning_zero_is_called_underpowered():
    mv = _classify(-0.03, -0.18, 0.11)
    assert not mv.excludes_zero
    assert "Underpowered" in describe(mv)


def test_no_price_is_quoted_when_a_bigger_design_would_not_help():
    """The arithmetic will happily solve for a design *smaller* than the one
    just run when the margin is wide. Quoting it would be absurd."""
    mv = classify(_result(-0.42, -0.57, -0.28), metric="pass rate", margin=0.30, n_trials=8)
    assert mv.requirement is None


def test_a_price_is_still_quoted_when_more_data_would_help():
    mv = classify(_result(0.0, -0.30, 0.30), metric="pass rate", margin=0.02, n_trials=5)
    assert mv.requirement is not None
    assert mv.requirement.n_runs > mv.result.n_tasks * 5 * 2


def test_a_missing_interval_does_not_claim_to_exclude_zero():
    assert not _classify(0.0, None, None).excludes_zero
