"""EvolutionLedger — SQLite persistence for runs, outcomes, reflexes, and promotions.

Mirrors the shape of ``DecisionLogger``:
  - idempotent ``initialize()`` — safe to call multiple times
  - never raises on failure — a broken ledger must never break the agent
  - optional ``db=None`` mode with in-memory fallback for tests
  - async locks around writes to avoid SQLite contention

Every table uses ``CREATE TABLE IF NOT EXISTS`` in ``_SCHEMA``; future
column additions go through ``_MIGRATIONS`` with try/except around each
ALTER so upgrades are lossless across runs.

Callers should treat the dataclasses in ``evolution.schemas`` as the
authoritative shape of persisted records. The ledger handles
JSON serialization for the blob columns (``env_fingerprint``, ``scope``,
``tools``, ``triggers``, ``invariants``, ``abort_conditions``,
``expected_mutations``, ``evidence_run_ids``, ``components``,
``baseline_ids``, ``evidence``, ``eligible_ids``, ``top_samples``).
"""

from __future__ import annotations

import asyncio
import json
import logging
from dataclasses import asdict
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any

from agntspace.evolution.bayes import ema_update, variance_update
from agntspace.evolution.schemas import (
    POLICY_INCUMBENT_DEFAULT,
    AntiReflex,
    OutcomeVector,
    Promotion,
    Reflex,
    ReflexPosterior,
    ReplayJob,
    RunManifest,
    SelectorTelemetry,
)

if TYPE_CHECKING:
    import aiosqlite

log = logging.getLogger(__name__)


class EvolutionLedger:
    """SQLite-backed store for evolution runs, outcomes, reflexes, and promotions."""

    _SCHEMA = """
    CREATE TABLE IF NOT EXISTS evolution_runs (
        run_id          TEXT PRIMARY KEY,
        correlation_id  TEXT NOT NULL,
        parent_run_id   TEXT DEFAULT '',
        caller          TEXT NOT NULL,
        trigger_type    TEXT NOT NULL,
        skill_name      TEXT NOT NULL,
        agent_name      TEXT NOT NULL,
        policy_id       TEXT NOT NULL DEFAULT 'incumbent_default',
        model_name      TEXT NOT NULL DEFAULT '',
        model_tier      TEXT NOT NULL DEFAULT '',
        env_fingerprint TEXT NOT NULL DEFAULT '{}',
        started_at      TEXT NOT NULL,
        completed_at    TEXT DEFAULT '',
        duration_ms     INTEGER DEFAULT 0
    );
    CREATE INDEX IF NOT EXISTS idx_evo_runs_skill       ON evolution_runs(skill_name, started_at);
    CREATE INDEX IF NOT EXISTS idx_evo_runs_policy      ON evolution_runs(policy_id, started_at);
    CREATE INDEX IF NOT EXISTS idx_evo_runs_correlation ON evolution_runs(correlation_id);
    CREATE INDEX IF NOT EXISTS idx_evo_runs_caller      ON evolution_runs(caller, started_at);

    CREATE TABLE IF NOT EXISTS evolution_outcomes (
        run_id          TEXT PRIMARY KEY,
        utility         REAL NOT NULL DEFAULT 0.0,
        zero_effect     INTEGER NOT NULL DEFAULT 0,
        blocked         INTEGER NOT NULL DEFAULT 0,
        components_json TEXT NOT NULL DEFAULT '{}',
        computed_at     TEXT NOT NULL
    );
    CREATE INDEX IF NOT EXISTS idx_evo_outcomes_utility ON evolution_outcomes(utility);

    CREATE TABLE IF NOT EXISTS evolution_reflexes (
        reflex_id              TEXT PRIMARY KEY,
        kind                   TEXT NOT NULL CHECK(kind IN ('reflex','anti')),
        name                   TEXT NOT NULL DEFAULT '',
        version                INTEGER NOT NULL DEFAULT 1,
        scope_json             TEXT NOT NULL DEFAULT '{}',
        preferred_agent        TEXT DEFAULT '',
        preferred_tier         TEXT DEFAULT '',
        tools_json             TEXT DEFAULT '[]',
        triggers_json          TEXT DEFAULT '[]',
        preconditions_json     TEXT DEFAULT '[]',
        invariants_json        TEXT DEFAULT '[]',
        abort_conditions_json  TEXT DEFAULT '[]',
        expected_mutations_json TEXT DEFAULT '{}',
        evidence_run_ids_json  TEXT DEFAULT '[]',
        status                 TEXT NOT NULL DEFAULT 'draft',
        created_at             TEXT NOT NULL,
        last_validated_at      TEXT DEFAULT '',
        decay_half_life_days   REAL NOT NULL DEFAULT 30.0,
        vault_path             TEXT DEFAULT ''
    );
    CREATE INDEX IF NOT EXISTS idx_evo_reflex_status ON evolution_reflexes(kind, status);

    CREATE TABLE IF NOT EXISTS evolution_reflex_posteriors (
        reflex_id    TEXT PRIMARY KEY,
        alpha        REAL NOT NULL DEFAULT 1.0,
        beta_param   REAL NOT NULL DEFAULT 1.0,
        utility_ema  REAL NOT NULL DEFAULT 0.0,
        utility_var  REAL NOT NULL DEFAULT 0.0,
        n_runs       INTEGER NOT NULL DEFAULT 0,
        last_run_at  TEXT DEFAULT ''
    );

    CREATE TABLE IF NOT EXISTS evolution_replay_jobs (
        job_id                  INTEGER PRIMARY KEY AUTOINCREMENT,
        incumbent_id            TEXT NOT NULL DEFAULT 'incumbent_default',
        candidate_id            TEXT NOT NULL,
        correlation_id_original TEXT NOT NULL DEFAULT '',
        correlation_id_replay   TEXT DEFAULT '',
        status                  TEXT NOT NULL DEFAULT 'pending',
        utility_delta           REAL DEFAULT NULL,
        created_at              TEXT NOT NULL,
        completed_at            TEXT DEFAULT ''
    );
    CREATE INDEX IF NOT EXISTS idx_evo_replay_status ON evolution_replay_jobs(status, candidate_id);

    CREATE TABLE IF NOT EXISTS evolution_promotions (
        promotion_id       TEXT PRIMARY KEY,
        candidate_id       TEXT NOT NULL,
        incumbent_id       TEXT NOT NULL DEFAULT 'incumbent_default',
        baseline_ids_json  TEXT DEFAULT '[]',
        evidence_json      TEXT NOT NULL DEFAULT '{}',
        decision           TEXT NOT NULL,
        confidence         REAL NOT NULL DEFAULT 0.0,
        rationale          TEXT NOT NULL DEFAULT '',
        created_at         TEXT NOT NULL,
        rollback_target    TEXT DEFAULT '',
        vault_path         TEXT DEFAULT ''
    );
    CREATE INDEX IF NOT EXISTS idx_evo_promotions_candidate ON evolution_promotions(candidate_id, created_at);

    CREATE TABLE IF NOT EXISTS evolution_selector_telemetry (
        run_id              TEXT PRIMARY KEY,
        policy_id           TEXT NOT NULL DEFAULT 'incumbent_default',
        eligible_ids_json   TEXT NOT NULL DEFAULT '[]',
        top_samples_json    TEXT NOT NULL DEFAULT '[]',
        chosen_reason       TEXT NOT NULL DEFAULT ''
    );
    """

    # Future-proofing: column additions go here with try/except around each one.
    # Empty at launch because all tables are new.
    _MIGRATIONS: list[str] = []  # arch:deterministic — SQL migrations list

    def __init__(self, db: aiosqlite.Connection | None = None) -> None:
        self._db = db
        self._lock = asyncio.Lock()
        self._initialized = False

        # In-memory fallback — used by tests that don't want a real DB.
        # Lists are append-only so test assertions can diff them.
        self._fb_runs: dict[str, RunManifest] = {}
        self._fb_outcomes: dict[str, OutcomeVector] = {}
        self._fb_reflexes: dict[str, Reflex] = {}
        self._fb_posteriors: dict[str, ReflexPosterior] = {}
        self._fb_replays: list[ReplayJob] = []
        self._fb_promotions: list[Promotion] = []
        self._fb_telemetry: dict[str, SelectorTelemetry] = {}

    # ── Schema ──

    async def initialize(self) -> None:
        if self._initialized or self._db is None:
            self._initialized = True
            return
        async with self._lock:
            try:
                await self._db.executescript(self._SCHEMA)
                for stmt in self._MIGRATIONS:
                    try:
                        await self._db.execute(stmt)
                    except Exception:
                        pass  # already applied
                await self._db.commit()
                self._initialized = True
            except Exception:
                log.exception("Failed to initialize evolution tables")

    # ── Runs ──

    async def persist_run(self, manifest: RunManifest) -> None:
        """Insert a new run manifest. Idempotent on (run_id) — re-persist overwrites."""
        if self._db is None:
            self._fb_runs[manifest.run_id] = manifest
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_runs "
                    "(run_id, correlation_id, parent_run_id, caller, trigger_type, "
                    " skill_name, agent_name, policy_id, model_name, model_tier, "
                    " env_fingerprint, started_at, completed_at, duration_ms) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                    (
                        manifest.run_id,
                        manifest.correlation_id,
                        manifest.parent_run_id,
                        manifest.caller,
                        manifest.trigger_type,
                        manifest.skill_name,
                        manifest.agent_name,
                        manifest.policy_id,
                        manifest.model_name,
                        manifest.model_tier,
                        json.dumps(manifest.env_fingerprint, default=str),
                        manifest.started_at,
                        manifest.completed_at,
                        manifest.duration_ms,
                    ),
                )
                await self._db.commit()
        except Exception:
            log.debug("persist_run failed", exc_info=True)

    async def complete_run(
        self, run_id: str, *, completed_at: str, duration_ms: int,
    ) -> None:
        """Mark a run as complete — updates completed_at + duration_ms."""
        if self._db is None:
            r = self._fb_runs.get(run_id)
            if r is not None:
                r.completed_at = completed_at
                r.duration_ms = duration_ms
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "UPDATE evolution_runs SET completed_at = ?, duration_ms = ? WHERE run_id = ?",
                    (completed_at, int(duration_ms), run_id),
                )
                await self._db.commit()
        except Exception:
            log.debug("complete_run failed", exc_info=True)

    async def get_run(self, run_id: str) -> dict[str, Any] | None:
        """Return a run + its outcome + its telemetry as a single dict, or None."""
        if self._db is None:
            m = self._fb_runs.get(run_id)
            if m is None:
                return None
            out = {**asdict(m)}
            if run_id in self._fb_outcomes:
                out["outcome"] = asdict(self._fb_outcomes[run_id])
            if run_id in self._fb_telemetry:
                out["telemetry"] = asdict(self._fb_telemetry[run_id])
            return out
        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT r.run_id, r.correlation_id, r.parent_run_id, r.caller, "
                "  r.trigger_type, r.skill_name, r.agent_name, r.policy_id, "
                "  r.model_name, r.model_tier, r.env_fingerprint, r.started_at, "
                "  r.completed_at, r.duration_ms, "
                "  o.utility, o.zero_effect, o.blocked, o.components_json, o.computed_at, "
                "  t.eligible_ids_json, t.top_samples_json, t.chosen_reason "
                "FROM evolution_runs r "
                "LEFT JOIN evolution_outcomes o ON o.run_id = r.run_id "
                "LEFT JOIN evolution_selector_telemetry t ON t.run_id = r.run_id "
                "WHERE r.run_id = ?",
                (run_id,),
            )
            row = await cur.fetchone()
        except Exception:
            log.debug("get_run failed", exc_info=True)
            return None
        if not row:
            return None
        return _row_to_run_dict(row)

    async def list_runs(
        self,
        *,
        skill_name: str | None = None,
        policy_id: str | None = None,
        caller: str | None = None,
        since: str | None = None,
        limit: int = 50,
        offset: int = 0,
    ) -> list[dict[str, Any]]:
        """Paginated list of recent runs (newest first) with joined outcome."""
        if self._db is None:
            runs = sorted(
                self._fb_runs.values(), key=lambda r: r.started_at, reverse=True,
            )
            if skill_name:
                runs = [r for r in runs if r.skill_name == skill_name]
            if policy_id:
                runs = [r for r in runs if r.policy_id == policy_id]
            if caller:
                runs = [r for r in runs if r.caller == caller]
            if since:
                runs = [r for r in runs if r.started_at >= since]
            out: list[dict[str, Any]] = []
            for r in runs[offset : offset + limit]:
                d = {**asdict(r)}
                if r.run_id in self._fb_outcomes:
                    d["outcome"] = asdict(self._fb_outcomes[r.run_id])
                out.append(d)
            return out

        if not self._initialized:
            await self.initialize()

        clauses: list[str] = []
        params: list[Any] = []
        if skill_name:
            clauses.append("r.skill_name = ?")
            params.append(skill_name)
        if policy_id:
            clauses.append("r.policy_id = ?")
            params.append(policy_id)
        if caller:
            clauses.append("r.caller = ?")
            params.append(caller)
        if since:
            clauses.append("r.started_at >= ?")
            params.append(since)
        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
        sql = (
            "SELECT r.run_id, r.correlation_id, r.parent_run_id, r.caller, "
            "  r.trigger_type, r.skill_name, r.agent_name, r.policy_id, "
            "  r.model_name, r.model_tier, r.env_fingerprint, r.started_at, "
            "  r.completed_at, r.duration_ms, "
            "  o.utility, o.zero_effect, o.blocked, o.components_json, o.computed_at, "
            "  t.eligible_ids_json, t.top_samples_json, t.chosen_reason "
            "FROM evolution_runs r "
            "LEFT JOIN evolution_outcomes o ON o.run_id = r.run_id "
            "LEFT JOIN evolution_selector_telemetry t ON t.run_id = r.run_id "
            f"{where} ORDER BY r.started_at DESC LIMIT ? OFFSET ?"
        )
        params.extend([int(limit), int(offset)])
        try:
            cur = await self._db.execute(sql, params)
            rows = await cur.fetchall()
        except Exception:
            log.debug("list_runs failed", exc_info=True)
            return []
        return [_row_to_run_dict(row) for row in rows]

    # ── Outcomes ──

    async def persist_outcome(self, outcome: OutcomeVector) -> None:
        """Insert or replace an outcome row by run_id."""
        if self._db is None:
            self._fb_outcomes[outcome.run_id] = outcome
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_outcomes "
                    "(run_id, utility, zero_effect, blocked, components_json, computed_at) "
                    "VALUES (?, ?, ?, ?, ?, ?)",
                    (
                        outcome.run_id,
                        float(outcome.utility),
                        1 if outcome.zero_effect else 0,
                        1 if outcome.blocked else 0,
                        json.dumps(outcome.components, default=str),
                        outcome.computed_at,
                    ),
                )
                await self._db.commit()
        except Exception:
            log.debug("persist_outcome failed", exc_info=True)

    # ── Reflexes ──

    async def save_reflex(self, reflex: Reflex | AntiReflex) -> None:
        """Upsert a reflex row. Ensures ``kind`` is set correctly."""
        kind = reflex.kind or "reflex"
        if self._db is None:
            self._fb_reflexes[reflex.reflex_id] = reflex
            # Create an initial posterior if missing
            if reflex.reflex_id not in self._fb_posteriors:
                self._fb_posteriors[reflex.reflex_id] = ReflexPosterior(reflex_id=reflex.reflex_id)
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_reflexes "
                    "(reflex_id, kind, name, version, scope_json, preferred_agent, "
                    " preferred_tier, tools_json, triggers_json, preconditions_json, "
                    " invariants_json, abort_conditions_json, expected_mutations_json, "
                    " evidence_run_ids_json, status, created_at, last_validated_at, "
                    " decay_half_life_days, vault_path) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                    (
                        reflex.reflex_id,
                        kind,
                        reflex.name,
                        int(reflex.version),
                        json.dumps(reflex.scope, default=str),
                        reflex.preferred_agent,
                        reflex.preferred_tier,
                        json.dumps(list(reflex.tools)),
                        json.dumps(list(reflex.triggers)),
                        json.dumps(list(reflex.preconditions)),
                        json.dumps(list(reflex.invariants)),
                        json.dumps(list(reflex.abort_conditions)),
                        json.dumps(reflex.expected_mutations, default=str),
                        json.dumps(list(reflex.evidence_run_ids)),
                        reflex.status,
                        reflex.created_at,
                        reflex.last_validated_at,
                        float(reflex.decay_half_life_days),
                        reflex.vault_path,
                    ),
                )
                # Make sure the posterior row exists for Phase 3+ selector reads
                await self._db.execute(
                    "INSERT OR IGNORE INTO evolution_reflex_posteriors "
                    "(reflex_id) VALUES (?)", (reflex.reflex_id,),
                )
                await self._db.commit()
        except Exception:
            log.debug("save_reflex failed", exc_info=True)

    async def get_reflex(self, reflex_id: str) -> Reflex | None:
        if self._db is None:
            return self._fb_reflexes.get(reflex_id)
        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT reflex_id, kind, name, version, scope_json, preferred_agent, "
                "preferred_tier, tools_json, triggers_json, preconditions_json, "
                "invariants_json, abort_conditions_json, expected_mutations_json, "
                "evidence_run_ids_json, status, created_at, last_validated_at, "
                "decay_half_life_days, vault_path "
                "FROM evolution_reflexes WHERE reflex_id = ?",
                (reflex_id,),
            )
            row = await cur.fetchone()
        except Exception:
            log.debug("get_reflex failed", exc_info=True)
            return None
        if not row:
            return None
        return _row_to_reflex(row)

    async def list_reflexes(
        self,
        *,
        kind: str | None = None,
        status: str | None = None,
        limit: int = 100,
    ) -> list[Reflex]:
        if self._db is None:
            rs = list(self._fb_reflexes.values())
            if kind:
                rs = [r for r in rs if r.kind == kind]
            if status:
                rs = [r for r in rs if r.status == status]
            return rs[: limit]
        if not self._initialized:
            await self.initialize()
        clauses: list[str] = []
        params: list[Any] = []
        if kind:
            clauses.append("kind = ?")
            params.append(kind)
        if status:
            clauses.append("status = ?")
            params.append(status)
        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
        try:
            cur = await self._db.execute(
                "SELECT reflex_id, kind, name, version, scope_json, preferred_agent, "
                "preferred_tier, tools_json, triggers_json, preconditions_json, "
                "invariants_json, abort_conditions_json, expected_mutations_json, "
                "evidence_run_ids_json, status, created_at, last_validated_at, "
                "decay_half_life_days, vault_path "
                f"FROM evolution_reflexes {where} ORDER BY created_at DESC LIMIT ?",
                (*params, int(limit)),
            )
            rows = await cur.fetchall()
        except Exception:
            log.debug("list_reflexes failed", exc_info=True)
            return []
        return [_row_to_reflex(row) for row in rows]

    async def update_reflex_status(
        self, reflex_id: str, status: str, *, last_validated_at: str = "",
    ) -> None:
        """Transition a reflex to a new lifecycle state."""
        if self._db is None:
            r = self._fb_reflexes.get(reflex_id)
            if r is not None:
                r.status = status
                if last_validated_at:
                    r.last_validated_at = last_validated_at
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                if last_validated_at:
                    await self._db.execute(
                        "UPDATE evolution_reflexes SET status = ?, last_validated_at = ? "
                        "WHERE reflex_id = ?",
                        (status, last_validated_at, reflex_id),
                    )
                else:
                    await self._db.execute(
                        "UPDATE evolution_reflexes SET status = ? WHERE reflex_id = ?",
                        (status, reflex_id),
                    )
                await self._db.commit()
        except Exception:
            log.debug("update_reflex_status failed", exc_info=True)

    # ── Posteriors ──

    async def get_posterior(self, reflex_id: str) -> ReflexPosterior:
        """Return posterior row — creates a default prior if missing."""
        if self._db is None:
            return self._fb_posteriors.setdefault(
                reflex_id, ReflexPosterior(reflex_id=reflex_id),
            )
        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT reflex_id, alpha, beta_param, utility_ema, utility_var, "
                "n_runs, last_run_at FROM evolution_reflex_posteriors WHERE reflex_id = ?",
                (reflex_id,),
            )
            row = await cur.fetchone()
        except Exception:
            log.debug("get_posterior failed", exc_info=True)
            return ReflexPosterior(reflex_id=reflex_id)
        if not row:
            # Seed a default prior
            await self._db.execute(
                "INSERT OR IGNORE INTO evolution_reflex_posteriors (reflex_id) VALUES (?)",
                (reflex_id,),
            )
            await self._db.commit()
            return ReflexPosterior(reflex_id=reflex_id)
        return ReflexPosterior(
            reflex_id=row[0], alpha=row[1], beta_param=row[2],
            utility_ema=row[3], utility_var=row[4],
            n_runs=row[5], last_run_at=row[6] or "",
        )

    async def update_posterior(
        self,
        reflex_id: str,
        utility: float,
        *,
        now: str | None = None,
        success_threshold: float = 0.75,
    ) -> ReflexPosterior:
        """Apply one observation to the reflex's Bayesian posterior.

        **Evolution v2 statistical-depth #1 (conjugate Beta-Bernoulli update,
        shipped 2026-04-24)**. Previously the update was the linearized
        approximation::

            alpha      += utility            # e.g. +0.83 for utility=0.83
            beta_param += (1 - utility)      # +0.17

        This treated utility as a partial success/failure weight, which
        gives the posterior MEAN approximately the right answer — but the
        posterior VARIANCE is miscalibrated because the update is not the
        true Beta conjugate for Bernoulli observations. Downstream
        ``prob_greater`` (promotion arena) and Thompson sampling both
        depend on the variance being correct.

        The fix: treat each run as a Bernoulli trial keyed on the
        ``success_threshold`` (default matches ``mining.success_utility_
        threshold`` in ``evolution.yaml`` — one notion of success, shared
        across miner + posterior). Beta(α, β) is the exact conjugate prior
        for Bernoulli(p) observations, so::

            if utility >= threshold:
                alpha += 1     # success
            else:
                beta_param += 1   # failure

        is the correct posterior update. Mean and variance both match
        scipy.stats.beta.{mean,var} for integer α, β. Locked by tests in
        ``tests/unit/evolution/test_conjugate_update.py``.

        Continuous utility is NOT discarded — the EMA + variance tracks
        continuous utility for effect-size reporting (Evolution v2 item
        #2) and for the Thompson-score scaling in ``select_policy``
        (``0.5 + 0.5 * utility_ema``). The Beta posterior and the EMA are
        deliberately two separate signals:
          - Beta(α, β): P(next run succeeds) — used by Thompson sampling
            and prob_greater gate. Bernoulli-conjugate update.
          - utility_ema: expected utility magnitude — used by reporting
            and the score-scaling heuristic. Continuous EMA update.

        **Backward compat**: existing posteriors on disk that were built
        with the linearized update still work — Beta(α, β) is valid for
        any α, β > 0 regardless of how the counts accumulated. Future
        updates from here integer-increment α or β, so the posterior
        gradually converges to the correct distribution over new runs.
        No migration needed.

        Args:
            reflex_id: row to update
            utility: observed utility in [0, 1] (clamped)
            now: optional ISO timestamp override (for tests)
            success_threshold: cutoff for Bernoulli classification.
                Caller (service.record_outcome / feedback) reads from
                ``config.mining.success_utility_threshold`` so binary
                classification shares ONE definition across the system.
        """
        now_iso = now or datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        post = await self.get_posterior(reflex_id)

        clamped_utility = max(0.0, min(1.0, float(utility)))
        # Guard against out-of-range thresholds — the caller typically
        # passes the YAML-sourced 0.75 but a misconfigured YAML shouldn't
        # collapse the posterior update to a trivial branch.
        threshold = max(0.0, min(1.0, float(success_threshold)))

        # Beta-Bernoulli conjugate update: one observation → +1 to α or β.
        if clamped_utility >= threshold:
            post.alpha += 1.0
        else:
            post.beta_param += 1.0

        # EMA + variance keep continuous utility (unchanged from v1) —
        # used by reporting + score scaling, NOT by Thompson sampling.
        new_ema = ema_update(post.utility_ema, clamped_utility)
        post.utility_var = variance_update(post.utility_var, clamped_utility, new_ema)
        post.utility_ema = new_ema
        post.n_runs += 1
        post.last_run_at = now_iso

        if self._db is None:
            self._fb_posteriors[reflex_id] = post
            return post
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_reflex_posteriors "
                    "(reflex_id, alpha, beta_param, utility_ema, utility_var, n_runs, last_run_at) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (
                        reflex_id, post.alpha, post.beta_param,
                        post.utility_ema, post.utility_var,
                        post.n_runs, post.last_run_at,
                    ),
                )
                await self._db.commit()
        except Exception:
            log.debug("update_posterior failed", exc_info=True)
        return post

    # ── Replay jobs ──

    async def enqueue_replay(
        self, *, incumbent_id: str, candidate_id: str, correlation_id_original: str,
    ) -> int:
        """Create a pending replay job. Returns job_id (0 in fallback mode)."""
        job = ReplayJob(
            incumbent_id=incumbent_id,
            candidate_id=candidate_id,
            correlation_id_original=correlation_id_original,
        )
        if self._db is None:
            job.job_id = len(self._fb_replays) + 1
            self._fb_replays.append(job)
            return job.job_id
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                cur = await self._db.execute(
                    "INSERT INTO evolution_replay_jobs "
                    "(incumbent_id, candidate_id, correlation_id_original, "
                    " correlation_id_replay, status, utility_delta, created_at, completed_at) "
                    "VALUES (?, ?, ?, ?, ?, NULL, ?, ?)",
                    (
                        incumbent_id, candidate_id, correlation_id_original,
                        job.correlation_id_replay, job.status,
                        job.created_at, job.completed_at,
                    ),
                )
                await self._db.commit()
                return cur.lastrowid or 0
        except Exception:
            log.debug("enqueue_replay failed", exc_info=True)
            return 0

    async def complete_replay(
        self, *, candidate_id: str, correlation_id_original: str,
        correlation_id_replay: str, utility_delta: float,
    ) -> bool:
        """Update a pending replay job to ``status='done'`` with the delta.

        Matches by ``(candidate_id, correlation_id_original)`` — if no
        pending row exists (e.g. a direct replay without enqueue), creates
        a new row in ``status='done'``. Returns True on success.
        """
        now_iso = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        if self._db is None:
            for j in self._fb_replays:
                if (j.candidate_id == candidate_id
                        and j.correlation_id_original == correlation_id_original
                        and j.status == "pending"):
                    j.status = "done"
                    j.correlation_id_replay = correlation_id_replay
                    j.utility_delta = utility_delta
                    j.completed_at = now_iso
                    return True
            # No pending row — synthesize a done row
            from agntspace.evolution.schemas import ReplayJob
            job = ReplayJob(
                job_id=len(self._fb_replays) + 1,
                incumbent_id=POLICY_INCUMBENT_DEFAULT,
                candidate_id=candidate_id,
                correlation_id_original=correlation_id_original,
                correlation_id_replay=correlation_id_replay,
                status="done",
                utility_delta=utility_delta,
                completed_at=now_iso,
            )
            self._fb_replays.append(job)
            return True

        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                # Try to update an existing pending row first
                cur = await self._db.execute(
                    "UPDATE evolution_replay_jobs SET status='done', "
                    "correlation_id_replay=?, utility_delta=?, completed_at=? "
                    "WHERE candidate_id=? AND correlation_id_original=? "
                    "AND status='pending'",
                    (correlation_id_replay, float(utility_delta), now_iso,
                     candidate_id, correlation_id_original),
                )
                if cur.rowcount == 0:
                    # No pending row — create one in 'done' state
                    await self._db.execute(
                        "INSERT INTO evolution_replay_jobs "
                        "(incumbent_id, candidate_id, correlation_id_original, "
                        " correlation_id_replay, status, utility_delta, "
                        " created_at, completed_at) "
                        "VALUES (?, ?, ?, ?, 'done', ?, ?, ?)",
                        (POLICY_INCUMBENT_DEFAULT, candidate_id,
                         correlation_id_original, correlation_id_replay,
                         float(utility_delta), now_iso, now_iso),
                    )
                await self._db.commit()
                return True
        except Exception:
            log.debug("complete_replay failed", exc_info=True)
            return False

    async def count_replays(self, candidate_id: str, *, status: str = "done") -> int:
        if self._db is None:
            return sum(
                1 for j in self._fb_replays
                if j.candidate_id == candidate_id and j.status == status
            )
        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT COUNT(*) FROM evolution_replay_jobs WHERE candidate_id = ? AND status = ?",
                (candidate_id, status),
            )
            row = await cur.fetchone()
            return int(row[0]) if row else 0
        except Exception:
            return 0

    # ── Promotions ──

    async def save_promotion(self, promotion: Promotion) -> None:
        if self._db is None:
            self._fb_promotions.append(promotion)
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_promotions "
                    "(promotion_id, candidate_id, incumbent_id, baseline_ids_json, "
                    " evidence_json, decision, confidence, rationale, created_at, "
                    " rollback_target, vault_path) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                    (
                        promotion.promotion_id,
                        promotion.candidate_id,
                        promotion.incumbent_id,
                        json.dumps(list(promotion.baseline_ids)),
                        json.dumps(promotion.evidence, default=str),
                        promotion.decision,
                        float(promotion.confidence),
                        promotion.rationale,
                        promotion.created_at,
                        promotion.rollback_target,
                        promotion.vault_path,
                    ),
                )
                await self._db.commit()
        except Exception:
            log.debug("save_promotion failed", exc_info=True)

    async def list_promotions(self, *, limit: int = 50) -> list[Promotion]:
        if self._db is None:
            return sorted(
                self._fb_promotions, key=lambda p: p.created_at, reverse=True,
            )[:limit]
        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT promotion_id, candidate_id, incumbent_id, baseline_ids_json, "
                "evidence_json, decision, confidence, rationale, created_at, "
                "rollback_target, vault_path "
                "FROM evolution_promotions ORDER BY created_at DESC LIMIT ?",
                (int(limit),),
            )
            rows = await cur.fetchall()
        except Exception:
            log.debug("list_promotions failed", exc_info=True)
            return []
        return [
            Promotion(
                promotion_id=row[0], candidate_id=row[1], incumbent_id=row[2],
                baseline_ids=json.loads(row[3] or "[]"),
                evidence=json.loads(row[4] or "{}"),
                decision=row[5], confidence=float(row[6] or 0.0),
                rationale=row[7] or "", created_at=row[8],
                rollback_target=row[9] or "", vault_path=row[10] or "",
            )
            for row in rows
        ]

    # ── Telemetry ──

    async def save_telemetry(self, telemetry: SelectorTelemetry) -> None:
        if self._db is None:
            self._fb_telemetry[telemetry.run_id] = telemetry
            return
        if not self._initialized:
            await self.initialize()
        try:
            async with self._lock:
                await self._db.execute(
                    "INSERT OR REPLACE INTO evolution_selector_telemetry "
                    "(run_id, policy_id, eligible_ids_json, top_samples_json, chosen_reason) "
                    "VALUES (?, ?, ?, ?, ?)",
                    (
                        telemetry.run_id,
                        telemetry.policy_id,
                        json.dumps(list(telemetry.eligible_ids)),
                        json.dumps(list(telemetry.top_samples), default=str),
                        telemetry.chosen_reason,
                    ),
                )
                await self._db.commit()
        except Exception:
            log.debug("save_telemetry failed", exc_info=True)

    # ── Reflex-scoped queries (for promotion arena + drift detection) ──

    async def list_runs_by_policy(
        self, policy_id: str, *, limit: int = 200,
    ) -> list[dict[str, Any]]:
        """Return recent runs + joined outcomes for a specific policy.

        Used by the PromotionArena (to evaluate candidate performance
        under live traffic) and DriftDetector (to check for regressions
        in an active reflex's recent performance window).
        """
        if self._db is None:
            runs = [
                r for r in sorted(
                    self._fb_runs.values(), key=lambda x: x.started_at, reverse=True,
                ) if r.policy_id == policy_id
            ]
            out: list[dict[str, Any]] = []
            for r in runs[:limit]:
                d = asdict(r)
                if r.run_id in self._fb_outcomes:
                    d["outcome"] = asdict(self._fb_outcomes[r.run_id])
                out.append(d)
            return out

        if not self._initialized:
            await self.initialize()
        try:
            cur = await self._db.execute(
                "SELECT r.run_id, r.correlation_id, r.parent_run_id, r.caller, "
                "  r.trigger_type, r.skill_name, r.agent_name, r.policy_id, "
                "  r.model_name, r.model_tier, r.env_fingerprint, r.started_at, "
                "  r.completed_at, r.duration_ms, "
                "  o.utility, o.zero_effect, o.blocked, o.components_json, o.computed_at "
                "FROM evolution_runs r "
                "LEFT JOIN evolution_outcomes o ON o.run_id = r.run_id "
                "WHERE r.policy_id = ? "
                "ORDER BY r.started_at DESC LIMIT ?",
                (policy_id, int(limit)),
            )
            rows = await cur.fetchall()
        except Exception:
            log.debug("list_runs_by_policy failed", exc_info=True)
            return []
        return [_row_to_run_dict(row[:19]) for row in rows]

    async def recent_safety_ema(self, policy_id: str, *, window: int = 20) -> float:
        """Return avg ``safety`` subscore across the last N runs of a policy.

        Used by the promotion arena's safety gate AND the drift detector's
        regression check. Returns 1.0 when there aren't enough runs yet —
        a new candidate gets the benefit of the doubt until real data exists,
        but the ``min_runs`` gate prevents premature promotion anyway.
        """
        runs = await self.list_runs_by_policy(policy_id, limit=window)
        safety_scores: list[float] = []
        for r in runs:
            outcome = r.get("outcome") or {}
            components = outcome.get("components") or {}
            if "safety" in components:
                try:
                    safety_scores.append(float(components["safety"]))
                except (TypeError, ValueError):
                    continue
        if not safety_scores:
            return 1.0
        return sum(safety_scores) / len(safety_scores)

    async def recent_zero_effect_rate(
        self, policy_id: str, *, window: int = 20,
    ) -> float:
        """Return the zero-effect rate across the last N runs of a policy.

        Used by the drift detector — rising zero-effect rate on an active
        reflex is a strong regression signal.
        """
        runs = await self.list_runs_by_policy(policy_id, limit=window)
        if not runs:
            return 0.0
        zero = sum(1 for r in runs if (r.get("outcome") or {}).get("zero_effect"))
        return zero / len(runs)

    # ── Aggregates ──

    async def counts(self) -> dict[str, int]:
        """Top-level counts for /api/evolution/status."""
        if self._db is None:
            active = sum(1 for r in self._fb_reflexes.values() if r.status == "active")
            candidates = sum(1 for r in self._fb_reflexes.values() if r.status == "candidate")
            quarantined = sum(1 for r in self._fb_reflexes.values() if r.status == "quarantined")
            retired = sum(1 for r in self._fb_reflexes.values() if r.status == "retired")
            drafts = sum(1 for r in self._fb_reflexes.values() if r.status == "draft")
            anti = sum(1 for r in self._fb_reflexes.values() if r.kind == "anti")
            return {
                "runs": len(self._fb_runs),
                "outcomes": len(self._fb_outcomes),
                "reflexes": sum(1 for r in self._fb_reflexes.values() if r.kind == "reflex"),
                "anti_reflexes": anti,
                "active": active,
                "candidates": candidates,
                "drafts": drafts,
                "quarantined": quarantined,
                "retired": retired,
                "promotions": len(self._fb_promotions),
                "replays": len(self._fb_replays),
            }
        if not self._initialized:
            await self.initialize()
        out = {
            "runs": 0, "outcomes": 0, "reflexes": 0, "anti_reflexes": 0,
            "active": 0, "candidates": 0, "drafts": 0,
            "quarantined": 0, "retired": 0,
            "promotions": 0, "replays": 0,
        }
        try:
            out["runs"] = await _scalar(self._db, "SELECT COUNT(*) FROM evolution_runs")
            out["outcomes"] = await _scalar(self._db, "SELECT COUNT(*) FROM evolution_outcomes")
            out["reflexes"] = await _scalar(
                self._db, "SELECT COUNT(*) FROM evolution_reflexes WHERE kind='reflex'",
            )
            out["anti_reflexes"] = await _scalar(
                self._db, "SELECT COUNT(*) FROM evolution_reflexes WHERE kind='anti'",
            )
            for status_name in ("active", "candidate", "draft", "quarantined", "retired"):
                key = "candidates" if status_name == "candidate" else (
                    "drafts" if status_name == "draft" else status_name
                )
                out[key] = await _scalar(
                    self._db,
                    "SELECT COUNT(*) FROM evolution_reflexes WHERE status = ?",
                    (status_name,),
                )
            out["promotions"] = await _scalar(self._db, "SELECT COUNT(*) FROM evolution_promotions")
            out["replays"] = await _scalar(self._db, "SELECT COUNT(*) FROM evolution_replay_jobs")
        except Exception:
            log.debug("ledger.counts failed", exc_info=True)
        return out


# ── Helpers ──


async def _scalar(db: aiosqlite.Connection, sql: str, params: tuple = ()) -> int:
    cur = await db.execute(sql, params)
    row = await cur.fetchone()
    return int(row[0]) if row and row[0] is not None else 0


def _row_to_run_dict(row: tuple) -> dict[str, Any]:
    """Shape the joined runs+outcomes+telemetry row into a dict."""
    env_fp = json.loads(row[10] or "{}")
    d = {
        "run_id": row[0],
        "correlation_id": row[1],
        "parent_run_id": row[2] or "",
        "caller": row[3],
        "trigger_type": row[4],
        "skill_name": row[5],
        "agent_name": row[6],
        "policy_id": row[7] or POLICY_INCUMBENT_DEFAULT,
        "model_name": row[8] or "",
        "model_tier": row[9] or "",
        "env_fingerprint": env_fp,
        "started_at": row[11],
        "completed_at": row[12] or "",
        "duration_ms": int(row[13] or 0),
    }
    # Outcome join — may be None if persist_outcome hasn't fired yet
    if len(row) > 14 and row[14] is not None:
        d["outcome"] = {
            "utility": float(row[14] or 0.0),
            "zero_effect": bool(row[15]),
            "blocked": bool(row[16]),
            "components": json.loads(row[17] or "{}"),
            "computed_at": row[18] or "",
        }
    # Telemetry join
    if len(row) > 19 and row[19] is not None:
        d["telemetry"] = {
            "eligible_ids": json.loads(row[19] or "[]"),
            "top_samples": json.loads(row[20] or "[]"),
            "chosen_reason": row[21] or "",
        }
    return d


def _row_to_reflex(row: tuple) -> Reflex:
    """Shape a reflex DB row into a dataclass."""
    return Reflex(
        reflex_id=row[0],
        kind=row[1],
        name=row[2] or "",
        version=int(row[3] or 1),
        scope=json.loads(row[4] or "{}"),
        preferred_agent=row[5] or "",
        preferred_tier=row[6] or "",
        tools=json.loads(row[7] or "[]"),
        triggers=json.loads(row[8] or "[]"),
        preconditions=json.loads(row[9] or "[]"),
        invariants=json.loads(row[10] or "[]"),
        abort_conditions=json.loads(row[11] or "[]"),
        expected_mutations=json.loads(row[12] or "{}"),
        evidence_run_ids=json.loads(row[13] or "[]"),
        status=row[14] or "draft",
        created_at=row[15],
        last_validated_at=row[16] or "",
        decay_half_life_days=float(row[17] or 30.0),
        vault_path=row[18] or "",
    )
