"""Tests for the analysis layer and the terminal report.

The analysis layer is tested on its numbers; the renderer is tested on the
promises docs/08-surfaces.md makes about what must and must not appear on
screen — no bare "not significant", no per-task p-values, an interval or an
explicit absence beside every headline figure.
"""

from __future__ import annotations

import io

import pytest
from rich.console import Console

from ablate.config.variant import ConfigVariant
from ablate.record import AssertionRecord, TelemetryRecord, TrialRecord
from ablate.report.analysis import analyze
from ablate.report.terminal import render_report
from ablate.stats import Verdict

ARMS = ("A", "B")


def _record(
    variant_id: str,
    task_id: str,
    passed: bool,
    *,
    replicate: int = 1,
    turns: int = 4,
    tokens: int = 400,
    cost: float | None = 0.1,
    model: str | None = "claude-sonnet-5",
    error: str | None = None,
) -> TrialRecord:
    return TrialRecord(
        trial_id=f"{task_id}-{variant_id}-{replicate}",
        run_id="run1",
        task_id=task_id,
        variant_id=variant_id,
        variant_hash="sha256:abc",
        replicate=replicate,
        agent="claude-code",
        model=model,
        started_at="2026-01-01T00:00:00Z",
        duration_s=1.0,
        telemetry=TelemetryRecord(
            input_tokens=tokens, output_tokens=0, turns=turns, cost_usd=cost
        ),
        assertions=[AssertionRecord(id="a", type="exit_code", passed=passed)],
        passed=passed,
        error=error,
    )


def _suite(n_tasks: int, a_passes: int, b_passes: int, n_trials: int = 5, **kw):
    records = []
    for i in range(n_tasks):
        for r in range(n_trials):
            records.append(_record("A", f"task-{i:02d}", r < a_passes, replicate=r, **kw))
            records.append(_record("B", f"task-{i:02d}", r < b_passes, replicate=r, **kw))
    return records


# --- estimand ---------------------------------------------------------------


def test_arm_values_and_delta_reconcile():
    """Displayed A − displayed B must equal Δ, or the report contradicts itself."""
    analysis = analyze(_suite(10, 4, 2), ARMS, seed=1)
    pass_rate = analysis.metric("pass_rate")
    a, b = pass_rate.arm_values["A"], pass_rate.arm_values["B"]
    assert a - b == pytest.approx(pass_rate.verdict.delta)
    assert a == pytest.approx(0.8)
    assert b == pytest.approx(0.4)


def test_analysis_requires_exactly_two_arms():
    with pytest.raises(ValueError, match="two arms"):
        analyze(_suite(8, 3, 2), ("A", "B", "C"))


def test_errored_trials_are_excluded_and_counted():
    records = _suite(8, 3, 2)
    records.append(_record("A", "task-00", False, replicate=9, error="agent timed out"))
    analysis = analyze(records, ARMS, seed=1)
    assert analysis.errors == 1
    assert any("errored" in w for w in analysis.warnings)


def test_trials_per_arm_is_measured_not_assumed():
    """A run cut short by the budget ceiling has fewer trials than requested."""
    records = _suite(8, 2, 1, n_trials=3)
    assert analyze(records, ARMS, seed=1).n_trials == 3


# --- metric direction -------------------------------------------------------


def test_token_metric_is_lower_is_better():
    """Arm A burning more tokens per turn is a regression for A."""
    records = []
    for i in range(10):
        for r in range(5):
            records.append(_record("A", f"t{i}", True, replicate=r, tokens=2000, turns=4))
            records.append(_record("B", f"t{i}", True, replicate=r, tokens=1000, turns=4))

    tokens = analyze(records, ARMS, seed=1).metric("tokens_per_turn")
    assert tokens.verdict.delta == pytest.approx(1.0)  # +100% relative
    assert tokens.verdict.verdict is Verdict.REGRESSION


def test_trials_with_zero_turns_are_skipped_for_rate_metrics():
    """A crashed trial reports 0 turns; dividing by it would be nonsense."""
    records = _suite(8, 3, 2)
    records.append(_record("A", "task-00", False, replicate=9, turns=0, tokens=0))
    analysis = analyze(records, ARMS, seed=1)
    assert analysis.metric("tokens_per_turn").verdict.delta is not None


# --- degenerate tasks -------------------------------------------------------


def test_degenerate_tasks_are_flagged_both_ways():
    records = _suite(6, 3, 2)
    records += [_record("A", "always", True, replicate=r) for r in range(5)]
    records += [_record("B", "always", True, replicate=r) for r in range(5)]
    records += [_record("A", "never", False, replicate=r) for r in range(5)]
    records += [_record("B", "never", False, replicate=r) for r in range(5)]

    rows = {t.task_id: t for t in analyze(records, ARMS, seed=1).tasks}
    assert rows["always"].degenerate == "all pass both arms"
    assert rows["never"].degenerate == "none pass either arm"
    assert rows["task-00"].degenerate is None


def test_degenerate_warning_quantifies_the_cost():
    records = _suite(6, 3, 2)
    records += [_record("A", "always", True, replicate=r) for r in range(5)]
    records += [_record("B", "always", True, replicate=r) for r in range(5)]

    analysis = analyze(records, ARMS, seed=1)
    warning = next(w for w in analysis.warnings if "no signal" in w or "effectively" in w)
    assert "effectively 6 tasks" in warning
    assert "MDE" in warning
    assert analysis.informative_tasks == 6


def test_degenerate_tasks_are_not_dropped_from_the_estimate():
    """Excluding tasks on the basis of their observed outcome is a forking path
    that biases Δ̂. They are flagged, never silently removed."""
    with_degenerate = _suite(8, 5, 5)  # every task degenerate: all pass both arms
    analysis = analyze(with_degenerate, ARMS, seed=1)
    assert len(analysis.tasks) == 8
    assert analysis.metric("pass_rate").verdict.delta == pytest.approx(0.0)


# --- guardrails -------------------------------------------------------------


def test_mixed_models_warn_loudly():
    """A model version bump moves the baseline; Δ across two is meaningless."""
    records = _suite(4, 3, 2, model="claude-sonnet-5")
    records += _suite(4, 3, 2, model="claude-opus-5")
    analysis = analyze(records, ARMS, seed=1)
    assert analysis.mixes_models
    assert any("model" in w for w in analysis.warnings)


def test_small_suite_yields_no_interval_and_says_so():
    """`ablate init` scaffolds three tasks — the most common real path."""
    analysis = analyze(_suite(3, 4, 2), ARMS, seed=1)
    pass_rate = analysis.metric("pass_rate")
    assert not pass_rate.verdict.has_ci
    assert pass_rate.verdict.verdict is Verdict.INCONCLUSIVE
    assert "at least 8" in pass_rate.verdict.result.ci_withheld


# --- rendering --------------------------------------------------------------


def _render(records, variants=None) -> str:
    buffer = io.StringIO()
    console = Console(file=buffer, width=100, no_color=True, legacy_windows=False)
    render_report(
        console,
        analyze(records, ARMS, run_id="run1", seed=7, agent="claude-code"),
        variants
        or [
            ConfigVariant("A", "# Title\n\nSome guidance here.\n", "CLAUDE.md"),
            ConfigVariant("B", None, "/dev/null"),
        ],
        command="ablate compare CLAUDE.md /dev/null --trials 5",
        version="0.2.0",
    )
    return buffer.getvalue()


def test_report_never_prints_a_bare_not_significant():
    output = _render(_suite(10, 3, 3))
    assert "not significant" not in output.lower()
    assert "no detectable difference" in output.lower()


def test_report_labels_per_task_rows_as_descriptive():
    output = _render(_suite(10, 4, 2))
    assert "descriptive only" in output
    assert "p-value" not in output.lower()
    assert "p <" not in output


def test_report_shows_an_interval_beside_every_metric():
    output = _render(_suite(10, 4, 2))
    assert output.count("95% CI") == 3  # pass rate, tokens, turns


def test_report_states_the_absence_when_there_is_no_interval():
    output = _render(_suite(3, 4, 2))
    assert "no interval" in output
    assert "INCONCLUSIVE" in output


def test_report_names_the_method():
    assert "paired cluster bootstrap" in _render(_suite(10, 4, 2))


def test_report_shows_config_token_cost():
    output = _render(_suite(10, 4, 2))
    assert "tokens/turn" in output
    assert "estimates" in output  # never presented as exact


def test_report_points_at_free_reanalysis():
    assert "ablate report --last" in _render(_suite(10, 4, 2))


def test_report_marks_a_partial_run():
    buffer = io.StringIO()
    console = Console(file=buffer, width=100, no_color=True)
    render_report(
        console,
        analyze(_suite(10, 4, 2), ARMS, seed=1),
        [ConfigVariant("A", "x", "CLAUDE.md"), ConfigVariant("B", None, "/dev/null")],
        command="",
        partial="61/80 trials completed, budget ceiling reached",
    )
    assert "PARTIAL" in buffer.getvalue()
