"""Core evaluation harness: runs each scenario against SWE-Bench instances.

Prerequisites
-------------
- Docker daemon running (SWE-Bench uses per-instance Docker images)
- ``swebench`` package installed: ``uv add swebench``
- HuggingFace datasets access (SWE-Bench Lite is public)
- ``claude`` CLI on PATH with a valid API key

Usage (low-level — prefer run.py for the CLI):
    from benchmarks.swe_bench.harness import Harness, HarnessConfig
    from benchmarks.swe_bench.scenarios import BERNSTEIN_SONNET

    cfg = HarnessConfig(results_dir=Path("benchmarks/swe_bench/results"))
    harness = Harness(cfg)
    summary = harness.run_scenario(BERNSTEIN_SONNET, limit=10)

Architecture
------------
For each SWE-Bench Lite instance the harness:
  1. Checks out the repo at the target commit inside a temp workspace.
  2. Runs each agent in the scenario's pipeline sequentially, passing the
     previous agent's output (plan / partial patch) to the next.
  3. Collects the final unified diff produced by the implementer agent.
  4. Applies the patch and runs the instance's test suite via the swebench
     evaluation harness (Docker).
  5. Records the result (resolved/failed/error) plus wall-time and cost.

Agent invocation
----------------
Each agent is invoked as a ``claude`` subprocess in non-interactive mode:

    claude --model <model> --effort <effort> \
           --print --output-format json \
           "$(cat <prompt_file>)"

The JSON stdout includes ``cost_usd`` and ``tokens`` fields that we capture
for metrics.  The implementer's final message is expected to contain a
fenced ```diff block that we extract as the patch.
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
import re
import subprocess
import tempfile
import time
from dataclasses import dataclass, field, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING, Any

from benchmarks.swe_bench.metrics import (
    AgentTrace,
    InstanceResult,
    InstanceStatus,
    ResultStore,
    ScenarioSummary,
    aggregate,
)

if TYPE_CHECKING:
    from benchmarks.swe_bench.scenarios import AgentRole, Scenario

logger = logging.getLogger(__name__)
_REPO_ROOT = Path(__file__).resolve().parents[2]

# ---------------------------------------------------------------------------
# Prompt templates
# ---------------------------------------------------------------------------

_ANALYST_PROMPT = """\
You are a senior software engineer. You have been given a GitHub issue and the
repository checked out at the base commit.

Issue:
{issue_text}

Repository: {repo} @ {base_commit}
Working directory: {workdir}

Your job:
1. Identify the root cause of the issue.
2. List the files most likely involved (at most 5).
3. Write a concise implementation plan (bullet points, ≤200 words) that an
   implementer can follow to fix the issue.

Output ONLY the plan — no code, no diff.
"""

_IMPLEMENTER_PROMPT = """\
You are a senior software engineer. Apply the following plan to fix the issue.

Issue:
{issue_text}

Implementation plan:
{plan}

Repository: {repo} @ {base_commit}
Working directory: {workdir}

Instructions:
- Edit the relevant files to implement the fix.
- After editing, output the complete unified diff of your changes using:
    git diff

Wrap the diff in a fenced block:
```diff
<diff here>
```

Do not add tests. Do not modify test files.
"""

_SOLO_IMPLEMENTER_PROMPT = """\
You are a senior software engineer. Fix the following GitHub issue.

Issue:
{issue_text}

Repository: {repo} @ {base_commit}
Working directory: {workdir}

Instructions:
- Edit the relevant files to implement the fix.
- After editing, output the complete unified diff of your changes using:
    git diff

Wrap the diff in a fenced block:
```diff
<diff here>
```

Do not add tests. Do not modify test files.
"""

_QA_PROMPT = """\
You are a senior QA engineer. Review the following diff for correctness.

Issue:
{issue_text}

Proposed diff:
{patch}

Respond with one of:
  APPROVED  — the diff looks correct and sufficient
  REJECTED: <brief reason>  — the diff is wrong or incomplete

One line only.
"""


# ---------------------------------------------------------------------------
# Harness configuration
# ---------------------------------------------------------------------------


@dataclass
class HarnessConfig:
    """Runtime configuration for the evaluation harness."""

    results_dir: Path
    dataset: str = "princeton-nlp/SWE-bench_Lite"
    split: str = "test"
    # Parallelism — keep at 1 for SWE-Bench (Docker is the bottleneck)
    workers: int = 1
    # Timeout per agent call in seconds
    agent_timeout_s: int = 300
    # Path to ``claude`` binary; None = resolve from PATH
    claude_bin: str | None = None
    # Additional env vars forwarded to agent subprocesses
    extra_env: dict[str, str] = field(default_factory=lambda: dict[str, str]())


# ---------------------------------------------------------------------------
# Agent runner
# ---------------------------------------------------------------------------


class AgentRunner:
    """Invokes a single Claude agent subprocess and captures output."""

    def __init__(self, cfg: HarnessConfig) -> None:
        self.cfg = cfg
        self._claude = cfg.claude_bin or "claude"

    def run(
        self,
        role: AgentRole,
        prompt: str,
        workdir: Path,
    ) -> AgentTrace:
        """Run one agent and return its trace.

        Args:
            role: The agent role definition (model, effort, cost).
            prompt: The full prompt string to send.
            workdir: Working directory for the subprocess (the repo clone).

        Returns:
            AgentTrace with timing, token usage, and patch flag.
        """
        model_map = {
            "haiku": "claude-haiku-4-5-20251001",
            "sonnet": "claude-sonnet-4-6",
            "opus": "claude-opus-4-6",
        }
        model_id = model_map.get(role.model, role.model)

        cmd = [
            self._claude,
            "--model",
            model_id,
            "--print",
            "--output-format",
            "json",
            "--dangerously-skip-permissions",
            prompt,
        ]

        env = {**os.environ, **self.cfg.extra_env}

        t0 = time.monotonic()
        try:
            proc = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
                cwd=str(workdir),
                timeout=self.cfg.agent_timeout_s,
                env=env,
            )
            wall_time = time.monotonic() - t0
        except subprocess.TimeoutExpired:
            wall_time = self.cfg.agent_timeout_s
            return AgentTrace(
                role=role.role,
                model=role.model,
                wall_time_s=wall_time,
                tokens_used=0,
                cost_usd=0.0,
                exit_code=124,
                patch_produced=False,
            )
        except FileNotFoundError:
            raise RuntimeError(
                f"claude binary not found at '{self._claude}'. Ensure the Claude Code CLI is installed and on PATH."
            ) from None

        # Parse JSON output for cost/token metadata
        tokens_used = 0
        cost_usd = 0.0
        output_text = ""

        if proc.returncode == 0 and proc.stdout.strip():
            try:
                data: dict[str, Any] = json.loads(proc.stdout)
                output_text = data.get("result", "") or data.get("content", "")
                tokens_used = int(data.get("total_tokens", data.get("tokens", 0)))
                cost_usd = float(data.get("cost_usd", 0.0))
            except (json.JSONDecodeError, TypeError, ValueError):
                # Fall back to raw stdout as text
                output_text = proc.stdout

        if not cost_usd and tokens_used:
            cost_usd = role.estimate_cost(tokens_used)

        patch_produced = bool(re.search(r"```diff\s*\n.+?```", output_text, re.DOTALL))

        return AgentTrace(
            role=role.role,
            model=role.model,
            wall_time_s=wall_time,
            tokens_used=tokens_used,
            cost_usd=cost_usd,
            exit_code=proc.returncode,
            patch_produced=patch_produced,
        )

    def run_and_capture_text(
        self,
        role: AgentRole,
        prompt: str,
        workdir: Path,
    ) -> tuple[AgentTrace, str]:
        """Run an agent and also return its raw text output."""
        model_map = {
            "haiku": "claude-haiku-4-5-20251001",
            "sonnet": "claude-sonnet-4-6",
            "opus": "claude-opus-4-6",
        }
        model_id = model_map.get(role.model, role.model)

        cmd = [
            self._claude,
            "--model",
            model_id,
            "--print",
            "--output-format",
            "json",
            "--dangerously-skip-permissions",
            prompt,
        ]

        env = {**os.environ, **self.cfg.extra_env}

        t0 = time.monotonic()
        try:
            proc = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
                cwd=str(workdir),
                timeout=self.cfg.agent_timeout_s,
                env=env,
            )
            wall_time = time.monotonic() - t0
        except subprocess.TimeoutExpired:
            wall_time = float(self.cfg.agent_timeout_s)
            trace = AgentTrace(
                role=role.role,
                model=role.model,
                wall_time_s=wall_time,
                tokens_used=0,
                cost_usd=0.0,
                exit_code=124,
                patch_produced=False,
            )
            return trace, ""
        except FileNotFoundError:
            raise RuntimeError(f"claude binary not found at '{self._claude}'.") from None

        tokens_used = 0
        cost_usd = 0.0
        output_text = ""

        if proc.returncode == 0 and proc.stdout.strip():
            try:
                data: dict[str, Any] = json.loads(proc.stdout)
                output_text = data.get("result", "") or data.get("content", "")
                tokens_used = int(data.get("total_tokens", data.get("tokens", 0)))
                cost_usd = float(data.get("cost_usd", 0.0))
            except (json.JSONDecodeError, TypeError, ValueError):
                output_text = proc.stdout
        else:
            output_text = proc.stdout + proc.stderr

        if not cost_usd and tokens_used:
            cost_usd = role.estimate_cost(tokens_used)

        patch_produced = bool(re.search(r"```diff\s*\n.+?```", output_text, re.DOTALL))

        trace = AgentTrace(
            role=role.role,
            model=role.model,
            wall_time_s=wall_time,
            tokens_used=tokens_used,
            cost_usd=cost_usd,
            exit_code=proc.returncode,
            patch_produced=patch_produced,
        )
        return trace, output_text


def _extract_patch(text: str) -> str:
    """Pull the first ```diff ... ``` block from agent output."""
    m = re.search(r"```diff\s*\n(.*?)```", text, re.DOTALL)
    return m.group(1).strip() if m else ""


# ---------------------------------------------------------------------------
# SWE-Bench integration helpers
# ---------------------------------------------------------------------------


def _load_lite_instances(dataset: str, split: str, limit: int | None) -> list[dict[str, Any]]:
    """Load SWE-Bench Lite instances via HuggingFace datasets.

    Raises ImportError if the ``datasets`` package is not installed.
    """
    try:
        from datasets import load_dataset  # type: ignore[import]
    except ImportError as exc:
        raise ImportError("The 'datasets' package is required. Install it with: uv add datasets swebench") from exc

    ds = load_dataset(dataset, split=split)  # pyright: ignore[reportUnknownVariableType]
    instances: list[dict[str, Any]] = list(ds)  # pyright: ignore[reportUnknownArgumentType]
    if limit is not None:
        instances = instances[:limit]
    return instances


def _clone_repo_at_commit(repo: str, commit: str, workdir: Path) -> None:
    """Shallow-clone a GitHub repo and check out a specific commit."""
    # repo is e.g. "django/django"
    url = f"https://github.com/{repo}.git"
    subprocess.run(
        ["git", "clone", "--depth=50", url, str(workdir)],
        check=True,
        capture_output=True,
    )
    subprocess.run(
        ["git", "checkout", commit],
        check=True,
        capture_output=True,
        cwd=str(workdir),
    )


def _apply_patch(patch: str, workdir: Path) -> bool:
    """Apply a unified diff to the working directory. Returns True on success."""
    if not patch.strip():
        return False
    proc = subprocess.run(
        ["git", "apply", "--check"],
        input=patch,
        text=True,
        capture_output=True,
        cwd=str(workdir),
    )
    if proc.returncode != 0:
        return False
    subprocess.run(
        ["git", "apply"],
        input=patch,
        text=True,
        check=True,
        cwd=str(workdir),
    )
    return True


def _run_tests_via_swebench(
    instance: dict[str, Any],
    workdir: Path,
) -> bool:
    """Run instance tests using the swebench evaluation framework.

    Requires Docker and the swebench package.
    Returns True iff all FAIL_TO_PASS tests now pass.
    """
    try:
        from swebench.harness.run_evaluation import main as swe_eval  # type: ignore[import]
    except ImportError as exc:
        raise ImportError("The 'swebench' package is required. Install it with: uv add swebench") from exc

    # swebench evaluation works by re-running the test command inside Docker.
    # We collect results via a temporary predictions file.
    with tempfile.TemporaryDirectory() as tmpdir:
        pred_path = Path(tmpdir) / "predictions.json"
        pred_path.write_text(
            json.dumps(
                [
                    {
                        "instance_id": instance["instance_id"],
                        "model_patch": _read_diff(workdir),
                        "model_name_or_path": "bernstein-harness",
                    }
                ]
            ),
            encoding="utf-8",
        )
        results_path = Path(tmpdir) / "results"
        results_path.mkdir()

        with contextlib.suppress(SystemExit):
            swe_eval(
                dataset_name=instance.get("_dataset", "princeton-nlp/SWE-bench_Lite"),
                split="test",
                predictions_path=str(pred_path),
                max_workers=1,
                force_rebuild=False,
                cache_level="env",
                run_id="bernstein-eval",
            )

        # Parse results
        result_file = results_path / "results.json"
        if result_file.exists():
            data = json.loads(result_file.read_text(encoding="utf-8"))
            resolved_ids: list[str] = data.get("resolved", [])
            return instance["instance_id"] in resolved_ids

    return False


def _read_diff(workdir: Path) -> str:
    """Read the current git diff of the workdir as a patch string."""
    result = subprocess.run(
        ["git", "diff", "HEAD"],
        capture_output=True,
        text=True,
        cwd=str(workdir),
    )
    return result.stdout


# ---------------------------------------------------------------------------
# Per-instance evaluation
# ---------------------------------------------------------------------------


def _run_solo_instance(
    instance: dict[str, Any],
    scenario: Scenario,
    runner: AgentRunner,
    workdir: Path,
) -> tuple[list[AgentTrace], str]:
    """Run a single-agent scenario on one instance."""
    role = scenario.agents[0]
    prompt = _SOLO_IMPLEMENTER_PROMPT.format(
        issue_text=instance.get("problem_statement", ""),
        repo=instance.get("repo", ""),
        base_commit=instance.get("base_commit", ""),
        workdir=str(workdir),
    )
    trace, text = runner.run_and_capture_text(role, prompt, workdir)
    patch = _extract_patch(text)
    return [trace], patch


def _run_bernstein_instance(
    instance: dict[str, Any],
    scenario: Scenario,
    runner: AgentRunner,
    workdir: Path,
) -> tuple[list[AgentTrace], str]:
    """Run a 3-agent Bernstein pipeline on one instance."""
    roles_by_name = {a.role: a for a in scenario.agents}
    traces: list[AgentTrace] = []

    # Stage 1: analyst → plan
    analyst_role = roles_by_name["analyst"]
    analyst_prompt = _ANALYST_PROMPT.format(
        issue_text=instance.get("problem_statement", ""),
        repo=instance.get("repo", ""),
        base_commit=instance.get("base_commit", ""),
        workdir=str(workdir),
    )
    analyst_trace, plan_text = runner.run_and_capture_text(analyst_role, analyst_prompt, workdir)
    traces.append(analyst_trace)

    if analyst_trace.exit_code != 0:
        return traces, ""

    # Stage 2: implementer → patch
    impl_role = roles_by_name["implementer"]
    impl_prompt = _IMPLEMENTER_PROMPT.format(
        issue_text=instance.get("problem_statement", ""),
        plan=plan_text,
        repo=instance.get("repo", ""),
        base_commit=instance.get("base_commit", ""),
        workdir=str(workdir),
    )
    impl_trace, impl_text = runner.run_and_capture_text(impl_role, impl_prompt, workdir)
    traces.append(impl_trace)
    patch = _extract_patch(impl_text)

    # Stage 3: QA review (advisory — doesn't block patch application)
    qa_role = roles_by_name.get("qa")
    if qa_role and patch:
        qa_prompt = _QA_PROMPT.format(
            issue_text=instance.get("problem_statement", ""),
            patch=patch,
        )
        qa_trace, qa_text = runner.run_and_capture_text(qa_role, qa_prompt, workdir)
        traces.append(qa_trace)
        # If QA explicitly rejects and implementer has low confidence, skip patch
        if qa_text.strip().upper().startswith("REJECTED") and not impl_trace.patch_produced:
            patch = ""

    return traces, patch


# ---------------------------------------------------------------------------
# Main Harness
# ---------------------------------------------------------------------------


class Harness:
    """Orchestrates the full SWE-Bench evaluation loop."""

    def __init__(self, cfg: HarnessConfig) -> None:
        self.cfg = cfg
        self.store = ResultStore(cfg.results_dir)
        self.runner = AgentRunner(cfg)

    def run_scenario(
        self,
        scenario: Scenario,
        instances: list[dict[str, Any]] | None = None,
        limit: int | None = None,
    ) -> ScenarioSummary:
        """Evaluate *scenario* on SWE-Bench Lite and return aggregated metrics.

        Args:
            scenario: The scenario configuration to evaluate.
            instances: Pre-loaded instances (skip dataset download if provided).
            limit: Maximum number of instances to evaluate.

        Returns:
            ScenarioSummary with resolve rate, cost, and timing.
        """
        if instances is None:
            logger.info("Loading SWE-Bench Lite from HuggingFace…")
            instances = _load_lite_instances(self.cfg.dataset, self.cfg.split, limit)
        elif limit is not None:
            instances = instances[:limit]

        logger.info("Running scenario '%s' on %d instances", scenario.name, len(instances))

        results: list[InstanceResult] = []
        for idx, instance in enumerate(instances, 1):
            iid = instance["instance_id"]

            # Resume support: skip already-evaluated instances
            if self.store.already_evaluated(scenario.name, iid):
                logger.debug("Skipping already-evaluated %s", iid)
                continue

            logger.info("[%d/%d] %s — %s", idx, len(instances), scenario.name, iid)
            result = self._evaluate_instance(instance, scenario)
            results.append(result)
            self.store.append(result)

        # Merge with any previously-saved results
        all_results = self.store.load(scenario.name)
        summary = _annotate_summary(
            aggregate(all_results),
            scenario=scenario,
            dataset=self.cfg.dataset,
            source_type="eval",
            verified=True,
            notes="Verified SWE-Bench eval artifact generated from Docker-backed harness output.",
        )
        self.store.save_summary(summary)
        return summary

    def mock_scenario(
        self,
        scenario: Scenario,
        n_instances: int = 300,
        *,
        seed: int = 42,
    ) -> ScenarioSummary:
        """Generate simulated results for *scenario* without real agent execution.

        Produces realistic per-instance JSONL results using seeded random draws
        so the numbers are reproducible.  Intended for CI smoke-tests and for
        demonstrating the thesis narrative before a full Docker-based run.

        Simulated metrics are tuned to match expected SWE-Bench Lite performance
        based on published benchmarks and the scaffolding thesis design:

        - solo-sonnet     ~23 %  $0.14/issue
        - solo-opus       ~35 %  $1.20/issue
        - bernstein-sonnet ~38 % $0.42/issue   ← thesis: cheaper AND better than Opus
        - bernstein-mixed  ~36 % $0.16/issue   ← thesis: near-Opus quality at 1/7 cost

        Args:
            scenario: Scenario configuration to simulate.
            n_instances: Number of synthetic instances to generate (default 300 = full Lite).
            seed: Random seed for reproducibility.

        Returns:
            ScenarioSummary with aggregated simulated metrics.
        """
        import random

        _MOCK_PARAMS: dict[str, dict[str, float]] = {
            "solo-sonnet": dict(
                resolve_rate=0.230,
                cost_mean=0.14,
                cost_std=0.05,
                time_mean=95.0,
                time_std=20.0,
                tokens_mean=9_500,
                tokens_std=2_000,
            ),
            "solo-opus": dict(
                resolve_rate=0.350,
                cost_mean=1.20,
                cost_std=0.40,
                time_mean=110.0,
                time_std=25.0,
                tokens_mean=16_000,
                tokens_std=3_500,
            ),
            "bernstein-sonnet": dict(
                resolve_rate=0.383,
                cost_mean=0.42,
                cost_std=0.12,
                time_mean=195.0,
                time_std=40.0,
                tokens_mean=28_000,
                tokens_std=5_000,
            ),
            "bernstein-mixed": dict(
                resolve_rate=0.360,
                cost_mean=0.16,
                cost_std=0.05,
                time_mean=175.0,
                time_std=35.0,
                tokens_mean=22_000,
                tokens_std=4_000,
            ),
        }

        params = _MOCK_PARAMS.get(
            scenario.name,
            dict(
                resolve_rate=0.25,
                cost_mean=0.20,
                cost_std=0.08,
                time_mean=120.0,
                time_std=25.0,
                tokens_mean=12_000,
                tokens_std=2_500,
            ),
        )

        rng = random.Random(seed)
        results: list[InstanceResult] = []

        for i in range(n_instances):
            iid = f"mock_{scenario.name}_{i + 1:04d}"
            resolved = rng.random() < params["resolve_rate"]
            cost = max(0.01, rng.gauss(params["cost_mean"], params["cost_std"]))
            wall_time = max(10.0, rng.gauss(params["time_mean"], params["time_std"]))
            tokens = max(100, int(rng.gauss(params["tokens_mean"], params["tokens_std"])))

            status: InstanceStatus = "resolved" if resolved else "failed"
            result = InstanceResult(
                instance_id=iid,
                scenario_name=scenario.name,
                status=status,
                resolved=resolved,
                wall_time_s=wall_time,
                total_tokens=tokens,
                total_cost_usd=cost,
            )
            results.append(result)
            self.store.append(result)

        summary = _annotate_summary(
            aggregate(results),
            scenario=scenario,
            dataset=self.cfg.dataset,
            source_type="mock",
            verified=False,
            notes="Synthetic preview artifact generated by `benchmarks/swe_bench/run.py mock`.",
        )
        self.store.save_summary(summary)
        logger.info(
            "Mock run '%s': %d/%d resolved (%.1f %%)  mean cost $%.4f",
            scenario.name,
            summary.resolved,
            summary.total_instances,
            summary.resolve_rate * 100,
            summary.mean_cost_per_instance_usd,
        )
        return summary

    def _evaluate_instance(
        self,
        instance: dict[str, Any],
        scenario: Scenario,
    ) -> InstanceResult:
        t0 = time.monotonic()
        iid = instance["instance_id"]

        with tempfile.TemporaryDirectory(prefix=f"bernstein_swe_{iid}_") as tmpdir:
            workdir = Path(tmpdir) / "repo"

            try:
                _clone_repo_at_commit(
                    instance["repo"],
                    instance["base_commit"],
                    workdir,
                )
            except subprocess.CalledProcessError as exc:
                wall = time.monotonic() - t0
                return InstanceResult(
                    instance_id=iid,
                    scenario_name=scenario.name,
                    status="error",
                    resolved=False,
                    wall_time_s=wall,
                    total_tokens=0,
                    total_cost_usd=0.0,
                    error_message=f"clone failed: {exc}",
                )

            # Run pipeline
            try:
                is_solo = scenario.agent_count == 1
                if is_solo:
                    traces, patch = _run_solo_instance(instance, scenario, self.runner, workdir)
                else:
                    traces, patch = _run_bernstein_instance(instance, scenario, self.runner, workdir)
            except Exception as exc:
                wall = time.monotonic() - t0
                logger.exception("Pipeline error on %s/%s", scenario.name, iid)
                return InstanceResult(
                    instance_id=iid,
                    scenario_name=scenario.name,
                    status="error",
                    resolved=False,
                    wall_time_s=wall,
                    total_tokens=0,
                    total_cost_usd=0.0,
                    error_message=str(exc),
                )

            total_tokens = sum(t.tokens_used for t in traces)
            total_cost = sum(t.cost_usd for t in traces)

            if not patch:
                wall = time.monotonic() - t0
                return InstanceResult(
                    instance_id=iid,
                    scenario_name=scenario.name,
                    status="failed",
                    resolved=False,
                    wall_time_s=wall,
                    total_tokens=total_tokens,
                    total_cost_usd=total_cost,
                    agent_traces=traces,
                    error_message="no patch produced",
                )

            # Apply patch
            patch_ok = _apply_patch(patch, workdir)
            if not patch_ok:
                wall = time.monotonic() - t0
                return InstanceResult(
                    instance_id=iid,
                    scenario_name=scenario.name,
                    status="failed",
                    resolved=False,
                    wall_time_s=wall,
                    total_tokens=total_tokens,
                    total_cost_usd=total_cost,
                    agent_traces=traces,
                    patch=patch,
                    error_message="patch did not apply cleanly",
                )

            # Evaluate with SWE-Bench test runner
            try:
                instance["_dataset"] = self.cfg.dataset
                resolved = _run_tests_via_swebench(instance, workdir)
            except ImportError as exc:
                # swebench not installed — record patch but skip test evaluation
                logger.warning("swebench not available, skipping test eval: %s", exc)
                resolved = False
                wall = time.monotonic() - t0
                return InstanceResult(
                    instance_id=iid,
                    scenario_name=scenario.name,
                    status="skipped",
                    resolved=False,
                    wall_time_s=wall,
                    total_tokens=total_tokens,
                    total_cost_usd=total_cost,
                    agent_traces=traces,
                    patch=patch,
                    error_message="swebench not installed",
                )
            except Exception:
                logger.exception("Test eval error on %s/%s", scenario.name, iid)
                resolved = False

            wall = time.monotonic() - t0
            return InstanceResult(
                instance_id=iid,
                scenario_name=scenario.name,
                status="resolved" if resolved else "failed",
                resolved=resolved,
                wall_time_s=wall,
                total_tokens=total_tokens,
                total_cost_usd=total_cost,
                agent_traces=traces,
                patch=patch,
            )


def _annotate_summary(
    summary: ScenarioSummary,
    *,
    scenario: Scenario,
    dataset: str,
    source_type: str,
    verified: bool,
    notes: str,
) -> ScenarioSummary:
    """Attach provenance metadata required for public benchmark publication."""
    return replace(
        summary,
        verified=verified,
        source_type=source_type,
        dataset=dataset,
        sample_size=summary.total_instances,
        run_at=datetime.now(UTC).isoformat(),
        commit_sha=_current_commit_sha(),
        scenarios=[scenario.name],
        model_family=_scenario_model_family(scenario),
        notes=notes,
    )


def _scenario_model_family(scenario: Scenario) -> str:
    """Return a compact model-family label for one scenario."""
    models = sorted({agent.model for agent in scenario.agents})
    return ", ".join(models)


def _current_commit_sha() -> str:
    """Return the current git commit SHA, or an empty string on failure."""
    result = subprocess.run(
        ["git", "rev-parse", "HEAD"],
        cwd=_REPO_ROOT,
        capture_output=True,
        text=True,
        check=False,
    )
    if result.returncode != 0:
        return ""
    return result.stdout.strip()
