"""Trust domain discovery — cluster undomained contract actions into proposals.

Rule 12 substrate for architectural gap #3 (2026-04-18). The
`domains.yaml` map is hand-authored today, meaning every new contract
action lands in `general` until a human remaps it. That's quiet
friction: the user rarely knows to do the remapping until behavior
feels off.

This service walks the ``decision_log`` SQLite table, finds undomained
actions, clusters them by co-occurrence (actions firing in the same
correlation chain) and lexical affix (shared prefix/suffix, e.g.
``github_*``). Clusters above the YAML thresholds become ``DomainProposal``
objects surfaced via ``GET /api/trust/domain-proposals`` for user review.

**The engine never writes to ``domains.yaml``** — every apply goes
through an explicit user approval endpoint that validates the YAML
shape, backs up the old file, and reloads the TrustService.

**Strategy in YAML** (``_meta/trust-domain-discovery/config/discovery.yaml``):
  * ``evidence_window_days`` — how far back the scan looks
  * ``min_cluster_size`` — minimum actions for a valid cluster
  * ``min_total_occurrences`` — minimum evidence count
  * ``min_distinct_correlations`` — minimum independent observations
  * ``co_occurrence_threshold`` — Jaccard floor for edges
  * ``min_lexical_affix_chars`` — shortest shared affix to count as a cluster signal
  * ``lexical_weight`` — balance between co-occurrence and lexical signals
  * ``proposal_cooldown_days`` — how long a dismissed proposal is suppressed
  * ``max_actions_per_proposal`` — cap per proposal for human reviewability
  * ``min_auto_name_chars`` — shortest affix that auto-names a domain
  * ``collision_suffix`` — appended when the auto-name collides with an existing domain

All numeric defaults in ``_FALLBACK_CONFIG`` are marked
``# arch:deterministic`` — identical shape to the shipped YAML so first
boot / corrupted YAML still produces a functioning discovery pass.

The engine is read-only against the decision log and stateless across
calls. Safe to invoke from the heartbeat, an API endpoint, or a CLI
command without synchronization. ``discover()`` is idempotent — same
log + same config = same proposals.
"""

from __future__ import annotations

import logging
import os
import re
from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

log = logging.getLogger(__name__)


# Values MUST mirror the shipped ``discovery.yaml`` defaults. If either
# changes without the other, the zero-config first-boot path silently
# diverges from post-config behavior.
_FALLBACK_CONFIG: dict[str, Any] = {  # arch:deterministic — fallback when YAML missing/malformed
    "evidence_window_days": 30,
    "min_cluster_size": 2,
    "min_total_occurrences": 5,
    "min_distinct_correlations": 3,
    "co_occurrence_threshold": 0.3,
    "min_lexical_affix_chars": 4,
    "lexical_weight": 0.3,
    "proposal_cooldown_days": 7,
    "max_actions_per_proposal": 8,
    "min_auto_name_chars": 3,
    "collision_suffix": " (auto-discovered)",
}


@dataclass
class DomainProposal:
    """A proposal to create a new trust domain from clustered actions.

    The engine emits one or more of these per discovery pass. They are
    inert — consumers read ``proposal.contract_actions`` and present to
    the user; applying is a separate endpoint that validates + writes
    YAML. Never auto-applied.
    """

    suggested_name: str
    contract_actions: list[str]
    total_occurrences: int
    distinct_correlations: int
    shared_affix: str = ""
    justification: str = ""
    # Pairwise co-occurrence counts for "why this cluster?" audit. Keys are
    # sorted action pairs ``"a|b"`` → count of shared correlation chains.
    co_occurrence_pairs: dict[str, int] = field(default_factory=dict)


@dataclass(frozen=True)
class DiscoveryConfig:
    """Typed view over the YAML config with validated defaults."""

    evidence_window_days: int = 30
    min_cluster_size: int = 2
    min_total_occurrences: int = 5
    min_distinct_correlations: int = 3
    co_occurrence_threshold: float = 0.3
    min_lexical_affix_chars: int = 4
    lexical_weight: float = 0.3
    proposal_cooldown_days: int = 7
    max_actions_per_proposal: int = 8
    min_auto_name_chars: int = 3
    collision_suffix: str = " (auto-discovered)"

    @classmethod
    def from_dict(cls, data: dict[str, Any] | None) -> DiscoveryConfig:
        """Parse YAML (or any dict-ish) into a typed config.

        Unknown keys are ignored. Known keys with wrong types fall back
        to the arch:deterministic default — the engine never crashes on
        a malformed YAML.
        """
        merged = {**_FALLBACK_CONFIG, **(data or {})}
        def _int(key: str) -> int:
            try:
                v = int(merged.get(key, _FALLBACK_CONFIG[key]))
                return max(0, v)  # never negative
            except (TypeError, ValueError):
                return int(_FALLBACK_CONFIG[key])
        def _float(key: str) -> float:
            try:
                v = float(merged.get(key, _FALLBACK_CONFIG[key]))
                return max(0.0, min(1.0 if key.endswith("_threshold") or key.endswith("_weight") else float("inf"), v))
            except (TypeError, ValueError):
                return float(_FALLBACK_CONFIG[key])
        def _str(key: str) -> str:
            v = merged.get(key, _FALLBACK_CONFIG[key])
            return str(v) if v is not None else str(_FALLBACK_CONFIG[key])

        return cls(
            evidence_window_days=_int("evidence_window_days"),
            min_cluster_size=max(1, _int("min_cluster_size")),
            min_total_occurrences=_int("min_total_occurrences"),
            min_distinct_correlations=_int("min_distinct_correlations"),
            co_occurrence_threshold=_float("co_occurrence_threshold"),
            min_lexical_affix_chars=_int("min_lexical_affix_chars"),
            lexical_weight=_float("lexical_weight"),
            proposal_cooldown_days=_int("proposal_cooldown_days"),
            max_actions_per_proposal=max(1, _int("max_actions_per_proposal")),
            min_auto_name_chars=_int("min_auto_name_chars"),
            collision_suffix=_str("collision_suffix"),
        )

    @classmethod
    def from_skills_dir(cls, skills_dir: Path | str | None) -> DiscoveryConfig:
        """Load config from the shipped skill YAML, falling back to defaults."""
        if not skills_dir:
            return cls.from_dict(None)
        path = Path(skills_dir) / "_meta" / "trust-domain-discovery" / "config" / "discovery.yaml"
        if not path.is_file():
            return cls.from_dict(None)
        try:
            import yaml
            data = yaml.safe_load(path.read_text(encoding="utf-8"))
            return cls.from_dict(data if isinstance(data, dict) else None)
        except Exception:
            log.warning(
                "Failed to load trust-domain-discovery config, using fallback",
                exc_info=True,
            )
            return cls.from_dict(None)


class TrustDomainDiscovery:
    """Clusters undomained contract actions into domain proposals.

    Usage:

        discovery = TrustDomainDiscovery(db, known_domain_actions)
        proposals = await discovery.discover()

    The engine is constructed with the set of actions that already have
    a domain (``known_domain_actions``) so the scan can skip them
    without re-reading the domain map. Callers can also pass a list of
    dismissed proposal signatures to enforce cooldown at the call site
    (see ``dismissed_signatures``).
    """

    def __init__(
        self,
        db: Any,
        *,
        known_domain_actions: set[str] | None = None,
        existing_domain_names: set[str] | None = None,
        config: DiscoveryConfig | None = None,
        dismissed_signatures: set[str] | None = None,
    ) -> None:
        self._db = db
        self._known = set(known_domain_actions or [])
        self._existing_names = set(existing_domain_names or [])
        self._config = config or DiscoveryConfig.from_dict(None)
        self._dismissed = set(dismissed_signatures or [])

    async def discover(self) -> list[DomainProposal]:
        """Scan the decision log and produce a list of proposals.

        Returns empty list if there's insufficient evidence, if the
        database is unavailable, or if every candidate cluster is
        currently suppressed by the cooldown list. Never raises.
        """
        try:
            rows = await self._fetch_undomained_rows()
        except Exception:
            log.warning("discovery: fetch failed", exc_info=True)
            return []
        if not rows:
            return []

        co_occurrence = self._build_co_occurrence(rows)
        action_counts = self._count_actions(rows)
        action_correlations = self._correlations_per_action(rows)

        clusters = self._cluster(co_occurrence, action_counts)
        proposals: list[DomainProposal] = []
        for cluster in clusters:
            if len(cluster) < self._config.min_cluster_size:
                continue
            total_occ = sum(action_counts.get(a, 0) for a in cluster)
            if total_occ < self._config.min_total_occurrences:
                continue
            distinct_chains: set[str] = set()
            for a in cluster:
                distinct_chains |= action_correlations.get(a, set())
            if len(distinct_chains) < self._config.min_distinct_correlations:
                continue

            # Split oversized clusters into chunks of max_actions_per_proposal.
            for sub in self._split_large_cluster(cluster):
                proposal = self._build_proposal(
                    cluster=sub,
                    action_counts=action_counts,
                    action_correlations=action_correlations,
                    co_occurrence=co_occurrence,
                )
                if self._signature(proposal) in self._dismissed:
                    continue
                proposals.append(proposal)

        # Sort by strongest evidence first (total occurrences desc).
        proposals.sort(key=lambda p: (-p.total_occurrences, p.suggested_name))
        return proposals

    # ── SQL layer ─────────────────────────────────────────────────────

    async def _fetch_undomained_rows(self) -> list[dict[str, str]]:
        """Return ``[{contract_action, correlation_id, ts}]`` for undomained
        decisions in the evidence window.

        Rows are de-duplicated on ``(contract_action, correlation_id)`` —
        a single correlation chain that fires the same action ten times
        counts as ONE co-occurrence observation. Without this, a tight
        retry loop would inflate a singleton into a false-positive cluster.
        """
        if self._db is None:
            return []

        since_iso = self._since_iso(self._config.evidence_window_days)
        # Avoid OFFSET — a simple filter is enough given the index on ts.
        # The query excludes contract_action='' (no action recorded, e.g.
        # a pure dispatch decision) and trust_domain='' — empty domain
        # means the decision predates Phase 5 and shouldn't be clustered.
        query = (
            "SELECT DISTINCT contract_action, correlation_id, ts "
            "FROM decision_log "
            "WHERE ts >= ? "
            "  AND contract_action != '' "
            "  AND COALESCE(correlation_id, '') != '' "
            "  AND (trust_domain = 'general' OR trust_domain = '') "
            "ORDER BY ts DESC"
        )
        try:
            cursor = await self._db.execute(query, (since_iso,))
            raw = await cursor.fetchall()
        except Exception:
            # In-memory fallback logger has no ``execute``; just degrade.
            log.debug("discovery: execute() failed, returning empty", exc_info=True)
            return []

        rows: list[dict[str, str]] = []
        seen_pairs: set[tuple[str, str]] = set()
        for r in raw:
            # aiosqlite returns sqlite3.Row-like tuples; index access works.
            action = str(r[0]) if r[0] else ""
            corr = str(r[1]) if r[1] else ""
            ts = str(r[2]) if r[2] else ""
            if not action or not corr:
                continue
            if action in self._known:
                # Safety net — the trust_domain filter above should catch these,
                # but the SQL can still return rows where domain was blank due
                # to a schema migration. Skip anything already mapped.
                continue
            pair = (action, corr)
            if pair in seen_pairs:
                continue
            seen_pairs.add(pair)
            rows.append({"contract_action": action, "correlation_id": corr, "ts": ts})
        return rows

    @staticmethod
    def _since_iso(days: int) -> str:
        """ISO-8601 UTC timestamp for ``now - days`` — used as the SQL cut."""
        from datetime import UTC, datetime, timedelta
        cutoff = datetime.now(UTC) - timedelta(days=max(0, int(days)))
        return cutoff.isoformat().replace("+00:00", "Z")

    # ── Feature extraction ────────────────────────────────────────────

    @staticmethod
    def _count_actions(rows: list[dict[str, str]]) -> dict[str, int]:
        """Map ``action → total distinct-chain occurrences``."""
        counts: dict[str, int] = defaultdict(int)
        for r in rows:
            counts[r["contract_action"]] += 1
        return dict(counts)

    @staticmethod
    def _correlations_per_action(rows: list[dict[str, str]]) -> dict[str, set[str]]:
        """Map ``action → set of correlation_ids it appeared in``."""
        by_action: dict[str, set[str]] = defaultdict(set)
        for r in rows:
            by_action[r["contract_action"]].add(r["correlation_id"])
        return dict(by_action)

    def _build_co_occurrence(
        self, rows: list[dict[str, str]]
    ) -> dict[tuple[str, str], float]:
        """Compute pairwise Jaccard similarity over correlation-chain membership.

        For every pair of actions (A, B), Jaccard(A, B) = |chains(A) ∩ chains(B)|
        / |chains(A) ∪ chains(B)|. Pairs below ``co_occurrence_threshold`` get
        dropped — no edge in the cluster graph.

        Lexical similarity bonus: if A and B share a prefix or suffix of at
        least ``min_lexical_affix_chars``, their Jaccard gets a
        ``lexical_weight`` bump (capped at 1.0). This lets clearly-related
        actions cluster even when their co-occurrence signal is weak.
        """
        correlations = self._correlations_per_action(rows)
        actions = sorted(correlations.keys())
        edges: dict[tuple[str, str], float] = {}
        for i, a in enumerate(actions):
            for b in actions[i + 1:]:
                chains_a = correlations[a]
                chains_b = correlations[b]
                inter = len(chains_a & chains_b)
                union = len(chains_a | chains_b)
                if union == 0:
                    continue
                jaccard = inter / union
                # Lexical affinity — shared prefix or suffix bumps score.
                affix = self._longest_shared_affix(a, b)
                if len(affix) >= self._config.min_lexical_affix_chars:
                    jaccard = min(
                        1.0,
                        jaccard * (1 - self._config.lexical_weight)
                        + 1.0 * self._config.lexical_weight,
                    )
                if jaccard >= self._config.co_occurrence_threshold:
                    edges[(a, b)] = jaccard
        return edges

    # ── Clustering ────────────────────────────────────────────────────

    def _cluster(
        self,
        edges: dict[tuple[str, str], float],
        action_counts: dict[str, int],
    ) -> list[list[str]]:
        """Union-find over the co-occurrence graph → list of action clusters.

        Connected components above ``min_cluster_size`` become proposals.
        Singletons are dropped — a single undomained action isn't a
        domain by itself.
        """
        parent: dict[str, str] = {}
        def find(x: str) -> str:
            while parent.get(x, x) != x:
                parent[x] = parent.get(parent[x], parent[x])  # path compression
                x = parent[x]
            return x
        def union(x: str, y: str) -> None:
            rx, ry = find(x), find(y)
            if rx != ry:
                parent[rx] = ry

        for a, b in edges:
            parent.setdefault(a, a)
            parent.setdefault(b, b)
            union(a, b)

        groups: dict[str, list[str]] = defaultdict(list)
        for node in parent:
            groups[find(node)].append(node)

        # Sort members alphabetically within each cluster for stable output.
        clusters = [sorted(members) for members in groups.values() if len(members) >= 2]
        # Sort clusters deterministically (largest first, then alpha).
        clusters.sort(key=lambda c: (-len(c), c[0] if c else ""))
        return clusters

    def _split_large_cluster(self, cluster: list[str]) -> list[list[str]]:
        """Split a cluster that exceeds max_actions_per_proposal into chunks.

        Simple partitioning — just chunk by index after sorting. A smarter
        split could preserve stronger co-occurrence pairs together, but
        the human reviewing the proposal can always approve a subset.
        """
        cap = self._config.max_actions_per_proposal
        if len(cluster) <= cap:
            return [cluster]
        chunks: list[list[str]] = []
        for i in range(0, len(cluster), cap):
            chunks.append(cluster[i:i + cap])
        return chunks

    # ── Proposal assembly ─────────────────────────────────────────────

    def _build_proposal(
        self,
        *,
        cluster: list[str],
        action_counts: dict[str, int],
        action_correlations: dict[str, set[str]],
        co_occurrence: dict[tuple[str, str], float],
    ) -> DomainProposal:
        affix = self._cluster_affix(cluster)
        suggested = self._suggest_name(cluster, affix)
        distinct: set[str] = set()
        for a in cluster:
            distinct |= action_correlations.get(a, set())
        total = sum(action_counts.get(a, 0) for a in cluster)

        pairs: dict[str, int] = {}
        for i, a in enumerate(cluster):
            for b in cluster[i + 1:]:
                key = f"{a}|{b}"
                jacc = co_occurrence.get((a, b)) or co_occurrence.get((b, a))
                if jacc is None:
                    continue
                chains_a = action_correlations.get(a, set())
                chains_b = action_correlations.get(b, set())
                pairs[key] = len(chains_a & chains_b)

        justification_parts: list[str] = [
            f"{len(cluster)} actions, {total} occurrences across {len(distinct)} correlation chains"
        ]
        if affix:
            justification_parts.append(f"shared affix '{affix}'")
        justification = "; ".join(justification_parts)

        return DomainProposal(
            suggested_name=suggested,
            contract_actions=cluster,
            total_occurrences=total,
            distinct_correlations=len(distinct),
            shared_affix=affix,
            justification=justification,
            co_occurrence_pairs=pairs,
        )

    def _suggest_name(self, cluster: list[str], affix: str) -> str:
        """Derive a human-readable domain name from the cluster.

        Priority:
            1. A shared affix of at least ``min_auto_name_chars``, cleaned
               of trailing underscores/punctuation → candidate.
            2. Fall back to ``"auto-domain-<sha8>"`` where the hash is
               deterministic over the sorted action list.

        If the candidate collides with an existing domain, append the
        configured ``collision_suffix`` so the user sees both options.
        """
        candidate = ""
        if affix and len(affix) >= self._config.min_auto_name_chars:
            candidate = self._clean_affix(affix)
        if not candidate:
            import hashlib
            seed = "|".join(sorted(cluster)).encode("utf-8")
            sha = hashlib.sha256(seed).hexdigest()[:8]
            candidate = f"auto-domain-{sha}"

        if candidate in self._existing_names:
            candidate = f"{candidate}{self._config.collision_suffix}"
        return candidate

    @staticmethod
    def _cluster_affix(cluster: list[str]) -> str:
        """Longest shared prefix OR suffix of every cluster member.

        Trimmed to the nearest word boundary so the affix is a clean word
        suitable for naming — ``read_email + send_email`` yields ``_email``
        raw, but we return ``email`` for ``suggested_name`` to emit. The
        raw version is still used by the clustering edge weight because
        a longer shared affix is more signal — trimming would artificially
        weaken borderline edges.
        """
        if not cluster:
            return ""
        if len(cluster) == 1:
            return ""
        prefix = os.path.commonprefix(cluster)
        reversed_items = [s[::-1] for s in cluster]
        suffix = os.path.commonprefix(reversed_items)[::-1]
        raw = prefix if len(prefix) >= len(suffix) else suffix
        return TrustDomainDiscovery._trim_to_word_boundary(raw, is_prefix=(raw == prefix))

    @staticmethod
    def _trim_to_word_boundary(affix: str, *, is_prefix: bool) -> str:
        """Trim a raw affix to the nearest word boundary.

        For a PREFIX like ``github_`` — already at a boundary, leave it.
        For a SUFFIX like ``d_email`` — the shared tail is 7 chars, but
        the clean word is ``email``. Find the first separator (``_`` / ``-``
        / ``.`` / ``/``) and return the content AFTER it.

        Leaves the affix alone when trimming would empty it or remove its
        distinguishing length; the goal is readable names, not aggressive
        shortening.
        """
        if not affix:
            return ""
        separators = r"[_\-./]"
        if is_prefix:
            # A prefix usually already ends at a boundary ("github_"). If
            # it doesn't, return as-is — the user can rename if desired.
            return affix
        # Suffix case — trim anything before the first separator.
        m = re.search(separators, affix)
        if m is None:
            return affix
        remainder = affix[m.end():]
        # Don't trim if it empties the affix or shortens below a few chars.
        if len(remainder) < 3:
            return affix
        return remainder

    @staticmethod
    def _longest_shared_affix(a: str, b: str) -> str:
        """Pairwise affix — longest shared prefix or suffix between two strings."""
        prefix = os.path.commonprefix([a, b])
        suffix = os.path.commonprefix([a[::-1], b[::-1]])[::-1]
        return prefix if len(prefix) >= len(suffix) else suffix

    @staticmethod
    def _clean_affix(affix: str) -> str:
        """Strip trailing connectors/punctuation so 'github_' becomes 'github'."""
        cleaned = re.sub(r"[_\-./]+$", "", affix).strip()
        cleaned = re.sub(r"^[_\-./]+", "", cleaned).strip()
        return cleaned

    @staticmethod
    def _signature(proposal: DomainProposal) -> str:
        """Stable identity for a proposal — used to match against the
        dismissed-cooldown set. Identical cluster membership → identical
        signature regardless of discovery timestamp."""
        return "|".join(sorted(proposal.contract_actions))
