"""Q2: is `--bare --append-system-prompt-file` equivalent to a real CLAUDE.md?

    python tools/q2_injection_equivalence.py --trials 5      # from a repo with .ablate/tasks

This is the experiment `docs/09-milestones.md` says must run in M1, and it is
still the largest unretired risk in the project. Everything `ablate` reports
rests on one substitution: because `--bare` skips `CLAUDE.md`, the config under
test is delivered as an appended system prompt instead. That is not
byte-identical to how Claude Code renders project memory — different position
in the prompt, different framing, not marked as memory.

**If the two differ materially, every number the tool produces is precise,
well-calibrated, and about the wrong thing.** No amount of care in the
statistics layer compensates for measuring a system-prompt appendix and calling
it your CLAUDE.md.

The experiment is a paired comparison the tool is already built to run, with
one twist: **both arms use the same config.** Only the delivery mechanism
differs.

    arm A   --bare + --append-system-prompt-file   (what ablate does)
    arm B   no --bare, config on disk              (what a user's session does)

A verdict of EQUIVALENT on pass rate *and* tokens is the result that licenses
the production path. Anything else is a finding worth acting on before launch.

## Reading the result

- **EQUIVALENT on both** — the substitution is faithful at this suite's
  resolution. Record the MDE alongside it: this says the difference is smaller
  than the design can see, not that it is zero.
- **REGRESSION or IMPROVEMENT on either** — the mechanisms are measurably
  different. `ablate` is measuring the appendix. Fix the injection before
  anything else ships.
- **INCONCLUSIVE** — the usual answer at small N, and not a pass. It means the
  experiment has not been run yet, however much it cost. The MDE printed
  underneath tells you what suite size would settle it.

## What this cannot control for

Arm B drops `--bare`, so ambient config — hooks, plugins, MCP servers, the
user's own auto-memory — can leak into it. That is unavoidable: a "natural"
session is defined by including those things. Run this in a clean checkout with
no user-level `~/.claude` customizations, or the difference you measure may be
your own dotfiles rather than the injection mechanism.
"""

from __future__ import annotations

import argparse
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path

from rich.console import Console

from ablate.agents.claude_code import INJECTION_APPEND, INJECTION_NATIVE, ClaudeCodeAdapter
from ablate.config.variant import ConfigVariant
from ablate.cost import SpendTracker, estimate_run, resolve_model
from ablate.errors import AblateError
from ablate.record import TrialRecord
from ablate.report.analysis import analyze
from ablate.report.terminal import render_report
from ablate.runner.schedule import build_jobs, run_jobs
from ablate.stats import Verdict, mde_pass_rate
from ablate.store.log import TrialLog
from ablate.task import load_tasks

console = Console()


def _repo_root() -> Path:
    out = subprocess.run(
        ["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True
    )
    return Path(out.stdout.strip())


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    parser.add_argument("config", nargs="?", default="CLAUDE.md")
    parser.add_argument("--tasks", default=".ablate/tasks")
    parser.add_argument("--trials", type=int, default=5)
    parser.add_argument("--concurrency", type=int, default=4)
    parser.add_argument("--max-spend", type=float, default=10.0)
    parser.add_argument("--model", default=None)
    parser.add_argument("--cheap", action="store_true")
    parser.add_argument("--margin", type=float, default=0.05)
    parser.add_argument("--seed", type=int, default=None)
    parser.add_argument("--yes", action="store_true")
    args = parser.parse_args()

    repo = _repo_root()
    config_path = repo / args.config
    if not config_path.exists():
        console.print(f"[red]{args.config} not found.[/red]")
        return 1

    try:
        tasks = load_tasks(repo / args.tasks)
    except AblateError as e:
        console.print(f"[red]{e}[/red]")
        return 1

    # The same bytes in both arms. Only the delivery differs, which is the
    # whole point — a difference in outcome cannot be attributed to the config.
    text = config_path.read_text()
    variants = [
        ConfigVariant(id="A", text=text, source=f"{args.config} (appended, --bare)"),
        ConfigVariant(id="B", text=text, source=f"{args.config} (on disk, native)"),
    ]

    price = resolve_model(args.model, cheap=args.cheap)
    adapters = {
        "A": ClaudeCodeAdapter(injection=INJECTION_APPEND),
        "B": ClaudeCodeAdapter(injection=INJECTION_NATIVE),
    }
    preflight = adapters["A"].preflight()
    if not preflight.ok:
        console.print(f"[red]claude-code preflight failed:[/red] {preflight.detail}")
        return 1

    total = len(tasks) * args.trials * 2
    estimate = estimate_run(total, price, concurrency=args.concurrency)

    console.print()
    console.print("  [bold]Q2 — injection equivalence[/bold]")
    console.print("  [dim]Same config in both arms. Only the delivery mechanism differs.[/dim]")
    console.print()
    console.print("  A  --bare + --append-system-prompt-file   [dim](what ablate does)[/dim]")
    console.print("  B  config on disk, no --bare              [dim](what a user gets)[/dim]")
    console.print()
    console.print(
        f"  {len(tasks)} tasks × {args.trials} trials × 2 arms = {total} runs on "
        f"{price.label} · ~${estimate.total_usd:.2f} · ~{estimate.minutes:.0f} min"
    )
    console.print(
        f"  [dim]This design resolves a difference of "
        f"{mde_pass_rate(len(tasks), args.trials) * 100:.0f}pp or larger. A smaller "
        f"one would not be visible, and EQUIVALENT means exactly that.[/dim]"
    )
    console.print()
    if len(tasks) < 8:
        console.print(
            "  [yellow]Fewer than 8 tasks — no confidence interval can be produced, "
            "so this run cannot settle Q2 either way.[/yellow]\n"
        )
    if estimate.total_usd > args.max_spend:
        console.print(
            f"  [red]Estimated ${estimate.total_usd:.2f} exceeds --max-spend "
            f"${args.max_spend:.2f}.[/red]"
        )
        return 1
    if not args.yes:
        console.print("  Run? [y/N] ", end="")
        if input().strip().lower() not in ("y", "yes"):
            return 1

    run_id = "q2-" + datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%SZ")
    run_dir = repo / ".ablate" / "runs" / run_id
    log = TrialLog(run_dir)
    tracker = SpendTracker(args.max_spend, total)

    records: list[TrialRecord] = []
    for arm, adapter in adapters.items():
        variant = next(v for v in variants if v.id == arm)
        produced, stopped = _run_arm(
            arm=arm,
            adapter=adapter,
            variant=variant,
            tasks=tasks,
            repo=repo,
            run_id=run_id,
            log=log,
            tracker=tracker,
            price=price,
            args=args,
        )
        records.extend(produced)
        if stopped:
            console.print("  [yellow]Spend ceiling reached — stopping.[/yellow]")
            break

    analysis = analyze(
        records,
        ("A", "B"),
        run_id=run_id,
        margin_overrides={"pass_rate": args.margin, "tokens_per_turn": args.margin},
        seed=args.seed,
        agent="claude-code",
        agent_version=preflight.version,
    )

    render_report(
        console=console,
        analysis=analysis,
        variants=variants,
        command="tools/q2_injection_equivalence.py",
        price=price,
        run_path=str((run_dir / "trials.jsonl").relative_to(repo)),
    )

    return _verdict(analysis)


def _run_arm(
    *,
    arm: str,
    adapter: ClaudeCodeAdapter,
    variant: ConfigVariant,
    tasks,  # noqa: ANN001 — list[Task]
    repo: Path,
    run_id: str,
    log: TrialLog,
    tracker: SpendTracker,
    price,  # noqa: ANN001 — ModelPrice
    args,  # noqa: ANN001 — argparse.Namespace
) -> tuple[list[TrialRecord], bool]:
    """One arm's trials.

    A function rather than a loop body so the progress callback closes over
    this arm's job count instead of a variable the next iteration rebinds.
    """
    jobs = build_jobs(tasks, [variant], args.trials, seed=args.seed)
    console.print(f"\n  [bold]arm {arm}[/bold] · {adapter.injection}")
    done = 0

    def on_result(record) -> None:  # noqa: ANN001 — TrialRecord
        nonlocal done
        done += 1
        log.append(record)
        tracker.record(record.telemetry.cost_usd)
        status = "[green]pass[/green]" if record.passed else "[red]fail[/red]"
        console.print(
            f"  [{done}/{len(jobs)}] {record.trial_id}  {status}  "
            f"[dim]${tracker.spent:.2f}[/dim]"
        )

    return run_jobs(
        jobs=jobs,
        repo=repo,
        worktrees_root=repo / ".ablate" / "worktrees",
        adapter=adapter,
        run_id=run_id,
        concurrency=args.concurrency,
        model=price.model_id,
        on_result=on_result,
        should_stop=lambda: tracker.ceiling is not None and tracker.spent >= tracker.ceiling,
    )


def _verdict(analysis) -> int:  # noqa: ANN001 — Analysis
    """Translate the comparison into an answer to Q2, and an exit code."""
    console.print("  [bold]Q2 VERDICT[/bold]\n")

    relevant = [analysis.metric("pass_rate"), analysis.metric("tokens_per_turn")]
    verdicts = [m.verdict.verdict for m in relevant if m]

    if any(v is Verdict.INCONCLUSIVE for v in verdicts):
        console.print(
            "    [yellow]UNSETTLED.[/yellow] At least one metric came back "
            "inconclusive, so this run does not answer Q2 — however much it cost. "
            "The interval is too wide to rule out a difference that would matter."
        )
        console.print(
            "    [dim]Add tasks (they buy more than trials do) and re-run.[/dim]\n"
        )
        return 2

    if all(v is Verdict.EQUIVALENT for v in verdicts):
        console.print(
            "    [green]FAITHFUL at this resolution.[/green] Delivering the config as "
            "an appended system prompt is indistinguishable from a real CLAUDE.md on "
            "both pass rate and token cost, within the margin."
        )
        console.print(
            "    [dim]Record the MDE with this result: it says the difference is "
            "smaller than this suite can see, not that it is zero.[/dim]\n"
        )
        return 0

    console.print(
        "    [red]NOT EQUIVALENT.[/red] The two delivery mechanisms produce "
        "measurably different behaviour, so `ablate` is currently measuring a "
        "system-prompt appendix and reporting it as your CLAUDE.md."
    )
    console.print(
        "    [dim]This is a correctness bug in the measurement, not a tuning "
        "issue. Fix the injection before anything else ships — and note that "
        "published results produced before this fix describe the appendix.[/dim]\n"
    )
    return 1


if __name__ == "__main__":
    sys.exit(main())
