"""File-type registry — one handler per file type, uniform output.

This is the frontier-shape contract that the whole ingest pipeline will
route through (decisions.md §2026-04-18, revised). Three concerns compose
per type:

    1. **Classify** — extensions, MIME types, magic-byte prefix, size cap,
       prompt_focus, default_source_type. Resolved from a FileTypeSpec.
    2. **Extract** — the registered extractor callable. Returns an
       ExtractResult carrying element-typed content (not just text),
       document-level metadata, embedded media refs, and provenance.
    3. **Render** — canvas_content_type + canvas_preview + ui_icon for
       downstream display. Consumed by Canvas and wiki surfaces.

The element-typed ExtractResult is load-bearing: downstream consumers (KB
compile, synthesize, graph population, canvas, multi-modal retrieval) will
add richer behaviour per element kind over time. Shipping text-only first
and "upgrading later" would require rewriting every consumer — doing it
right once is cheaper.

**Phase 1 goals (this commit)**:
- Define the dataclasses + registry shape.
- Port the 7 existing handlers as bundled registry entries (some emit a
  single Paragraph element — richness added per type in Phase 2+).
- Wire `_classify_source` + the three scattered extraction branches in
  `kb/service.py` through the registry.
- Emit provenance triples so later re-extraction decisions are queryable.

**Not in scope (later phases)**:
- Multi-extractor ladders per type (Phase 4).
- Multi-modal join with vision on embedded images (Phase 5).
- Canvas renderer registry + inverse rendering (Phase 6).
"""

from __future__ import annotations

import logging
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal

log = logging.getLogger(__name__)


# ── Element taxonomy ────────────────────────────────────────────────────

ElementKind = Literal[
    "title",           # Document title, single per doc by convention
    "heading",         # Section heading at any level (depth in attrs)
    "paragraph",       # Default narrative text element
    "table",           # Tabular data — rows/cols preserved in attrs
    "list_item",       # Single list entry (ordered or unordered)
    "code",            # Code block — language in attrs
    "equation",        # LaTeX / MathML expression
    "image",           # Embedded image reference (path in attrs)
    "captioned_media", # Audio/video segment with timestamp + caption
]


@dataclass
class Element:
    """One structural piece of an extracted document.

    Text-only extractors emit a single `paragraph` element containing the
    whole document body. Richer extractors (PDF with layout, DOCX, EPUB)
    emit multiple elements preserving structure.

    Attributes are open-ended per kind so extractors can carry format-specific
    hints without us pre-declaring every possible field. Stable keys by kind:

    - `heading`: `{depth: 1-6}`
    - `table`:   `{rows: list[list[str]], headers: list[str] | None}`
    - `list_item`: `{depth: int, ordered: bool, marker: str | None}`
    - `code`:    `{language: str | None}`
    - `image`:   `{path: str, alt: str | None, caption: str | None}`
    - `captioned_media`: `{start_ms: int, end_ms: int, kind: "audio" | "video"}`

    All elements may carry:
    - `page: int | None` — source page number (PDF, EPUB)
    - `bbox: tuple[int,int,int,int] | None` — bounding box on the page
    - `parent_id: str | None` — id of enclosing element (nesting)
    - `element_id: str | None` — stable id for cross-references
    """

    kind: ElementKind
    text: str
    attrs: dict[str, Any] = field(default_factory=dict)
    page: int | None = None
    bbox: tuple[int, int, int, int] | None = None
    parent_id: str | None = None
    element_id: str | None = None


# ── Media references (embedded assets) ──────────────────────────────────

@dataclass
class MediaRef:
    """Pointer to a binary asset referenced by a document.

    Kept separate from Elements because media are extracted separately
    (copied to disk or resolved to URLs) and may be processed by different
    pipelines (vision, transcription). Each MediaRef usually corresponds
    to an Image or CaptionedMedia element in the elements list.
    """

    path: str               # vault-relative path or remote URL
    kind: Literal["image", "audio", "video"]
    mime: str | None = None
    size_bytes: int | None = None
    element_id: str | None = None  # back-ref to owning element


# ── Extraction result (the frontier contract) ───────────────────────────

@dataclass
class ExtractResult:
    """What every extractor returns.

    Downstream (kb/service.py, compile, canvas, graph) may read either:
    - `.as_text()` — concatenated text for legacy LLM prompting
    - `.elements` — structured content for richer consumers

    Provenance is captured at extract time so re-ingest decisions can be
    made later ("this was extracted with pdfplumber@0.10 before tables
    worked — worth re-ingesting now that we have unstructured_hi_res").
    """

    elements: list[Element]
    doc_metadata: dict[str, Any] = field(default_factory=dict)
    media: list[MediaRef] = field(default_factory=list)
    provenance: dict[str, Any] = field(default_factory=dict)

    def as_text(self, joiner: str = "\n\n") -> str:
        """Concatenate element text for legacy string-based consumers."""
        return joiner.join(e.text for e in self.elements if e.text)

    @property
    def is_empty(self) -> bool:
        return not any(e.text.strip() for e in self.elements)


# ── Extractor binding (Phase 4: multi-extractor ladder) ────────────────


CostTier = Literal["fast", "capable", "reasoning"]
Fidelity = Literal["low", "medium", "high"]


@dataclass
class ExtractorBinding:
    """One entry in a FileTypeSpec's extractor ladder.

    A spec can list multiple extractors in order of ascending fidelity so
    the ExtractionSelector can pick the right tier for the caller's budget
    (fast pass for bulk ingest, capable / reasoning tier for one-off
    re-ingest when better quality is worth the cost).

    Ladder ordering convention: **low→high fidelity**. The selector walks
    the list and picks the highest fidelity that fits the budget, so put
    `fast` / `low` entries first.

    Each binding names a registered extractor id (``pdfplumber`` in Phase
    1-3, ``pdfplumber_ocr`` from Phase 4) + the cost/fidelity metadata the
    selector uses. ``options`` are per-binding overrides passed to the
    extractor via context (take precedence over ``spec.extractor_options``).
    """

    extractor: str                                  # id in FileTypeRegistry._extractors
    cost_tier: CostTier = "fast"                    # mirrors ModelRouter's tier vocab
    fidelity: Fidelity = "medium"                   # low/medium/high expected quality
    description: str = ""                           # human-readable (shown in UI)
    options: dict[str, Any] = field(default_factory=dict)


# ── File type spec (one per supported format) ──────────────────────────

@dataclass
class FileTypeSpec:
    """Declares how one file type is classified, extracted, and rendered.

    Initial registry entries are built programmatically (Phase 1). Phase 2
    migrates these to YAML at `defaults/skills/file-types/{name}/config/file-type.yaml`
    discovered through SkillLoader. Phase 4 adds the optional ``extractors``
    ladder for multi-tier extraction.
    """

    # Identity
    type_name: str
    display_name: str

    # — Classify layer —
    extensions: tuple[str, ...]                 # ".pdf", ".docx" — lowercased
    mime_types: tuple[str, ...] = ()            # optional MIME matches
    magic_prefix: bytes | None = None           # content-sniffing fallback
    default_source_type: str = "article"        # inherits kb-ingest taxonomy
    prompt_focus: str = "Extract main points, key facts, and context."
    max_bytes: int = 10 * 1024 * 1024           # per-type size cap (watcher consults this)
    requires_binary: bool = False               # True = don't read as utf-8

    # — Extract layer —
    # Phase 1-3 shape: a single extractor id drives behaviour.
    extractor: str = "text_utf8"                # id into FileTypeRegistry._extractors
    extractor_options: dict[str, Any] = field(default_factory=dict)
    # Phase 4 shape: optional ladder. When non-empty, the registry's
    # `extract()` consults the ExtractionSelector; when empty, falls back
    # to the single `extractor` above (full backward compat with Phase 1-3).
    extractors: tuple[ExtractorBinding, ...] = ()
    chunking_strategy: Literal["fixed", "structural", "semantic"] = "fixed"
    transcribe: bool = False                    # audio/video → whisper
    vision_capable: bool = False                # images → multimodal LLM

    # — Render layer (informs Canvas + wiki) —
    canvas_content_type: str = "markdown"
    canvas_preview: Literal["inline", "thumbnail", "link-only"] = "inline"
    ui_icon: str = "FileText"                   # lucide icon name
    thumbnail_generator: str | None = None

    # ── Ladder helpers ─────────────────────────────────────────────────

    def has_ladder(self) -> bool:
        """True when this spec declares multiple extractors."""
        return len(self.extractors) > 0

    def default_binding(self) -> ExtractorBinding:
        """Return the binding the selector uses when no explicit tier/id
        is requested. For ladder-enabled specs, it's the FIRST entry (by
        convention, the lowest-cost tier — `fast`). For Phase 1-3 specs
        it's a synthetic binding over the single ``extractor`` field.
        """
        if self.extractors:
            return self.extractors[0]
        return ExtractorBinding(
            extractor=self.extractor,
            cost_tier="fast",
            fidelity="medium",
            options=dict(self.extractor_options or {}),
        )

    def find_binding(self, extractor_id: str) -> ExtractorBinding | None:
        """Look up a specific binding by extractor id. Returns None when
        the id isn't declared in this spec's ladder."""
        for b in self.extractors:
            if b.extractor == extractor_id:
                return b
        # Fallback: if the caller asks for the spec's single `extractor`
        # (Phase 1-3 shape), synthesize a binding for compat.
        if extractor_id == self.extractor:
            return ExtractorBinding(
                extractor=self.extractor,
                cost_tier="fast",
                fidelity="medium",
                options=dict(self.extractor_options or {}),
            )
        return None


# ── Extractor callable contract ─────────────────────────────────────────

# Extractors receive (path, spec, context) and return an ExtractResult.
# `context` carries runtime deps the extractor may need (llm provider for
# vision/transcribe, vault paths for sibling-image resolution, options).
ExtractorContext = dict[str, Any]
Extractor = Callable[[Path, FileTypeSpec, ExtractorContext], Awaitable[ExtractResult]]


class FileTypeError(Exception):
    """Raised when a file type can't be resolved or extracted."""


# ── The registry ────────────────────────────────────────────────────────

class FileTypeRegistry:
    """Single source of truth for file-type classification + extraction.

    Populated at startup from bundled defaults (Phase 1) and — later —
    from skill YAMLs and plugin hooks.

    The registry is *inert* when initialized — no extractor is imported
    until its type is actually requested. This keeps boot fast and lets
    us ship formats with heavy deps (python-docx, ebooklib, whisper)
    without bloating the install for users who don't need them.
    """

    def __init__(self) -> None:
        self._specs: dict[str, FileTypeSpec] = {}
        self._by_ext: dict[str, str] = {}        # ".pdf" → type_name
        self._by_mime: dict[str, str] = {}       # mime → type_name
        self._by_magic: list[tuple[bytes, str]] = []  # (prefix, type_name) ordered
        self._extractors: dict[str, Extractor] = {}

    # ── Registration ────────────────────────────────────────────────

    def register_type(self, spec: FileTypeSpec, *, overwrite: bool = False) -> None:
        """Add a FileTypeSpec. Raises if type_name already registered unless
        `overwrite=True` (plugins opting into override semantics).
        """
        if spec.type_name in self._specs and not overwrite:
            raise FileTypeError(
                f"file type '{spec.type_name}' already registered "
                f"(set overwrite=True to replace)"
            )
        self._specs[spec.type_name] = spec
        for ext in spec.extensions:
            ext_lc = ext.lower()
            prior = self._by_ext.get(ext_lc)
            if prior and prior != spec.type_name and not overwrite:
                log.warning(
                    "file-type extension collision: '%s' claimed by both '%s' and '%s' "
                    "(keeping '%s' — first wins; plugins must pass overwrite=True)",
                    ext_lc, prior, spec.type_name, prior,
                )
                continue
            self._by_ext[ext_lc] = spec.type_name
        for mime in spec.mime_types:
            self._by_mime[mime.lower()] = spec.type_name
        if spec.magic_prefix:
            self._by_magic.append((spec.magic_prefix, spec.type_name))
        log.debug("FileTypeRegistry: registered '%s' (exts=%s)",
                  spec.type_name, list(spec.extensions))

    def register_extractor(self, extractor_id: str, fn: Extractor) -> None:
        """Attach an extractor callable by id. Multiple types may share an
        extractor (e.g. generic `text_utf8` serving md + txt + rst).
        """
        self._extractors[extractor_id] = fn

    # ── Resolution ─────────────────────────────────────────────────

    def resolve_type(self, path: Path, mime: str | None = None) -> FileTypeSpec | None:
        """Find the spec for a file. Order: extension → MIME → magic bytes.

        Returns None when the file's type is unknown — caller decides
        whether to park it in `library/unsupported/` or fall through to
        a default text-handler (see kb/service.py's watcher logic).
        """
        ext = path.suffix.lower()
        if ext and ext in self._by_ext:
            return self._specs[self._by_ext[ext]]
        if mime:
            type_name = self._by_mime.get(mime.lower())
            if type_name:
                return self._specs[type_name]
        if self._by_magic:
            try:
                with path.open("rb") as f:
                    head = f.read(16)
                for prefix, type_name in self._by_magic:
                    if head.startswith(prefix):
                        return self._specs[type_name]
            except OSError:
                pass
        return None

    def get_spec(self, type_name: str) -> FileTypeSpec | None:
        return self._specs.get(type_name)

    def all_types(self) -> list[FileTypeSpec]:
        return list(self._specs.values())

    def all_extensions(self) -> frozenset[str]:
        return frozenset(self._by_ext.keys())

    # ── Extraction ─────────────────────────────────────────────────

    async def extract(
        self,
        path: Path,
        spec: FileTypeSpec,
        context: ExtractorContext | None = None,
        *,
        extractor_id: str | None = None,
    ) -> ExtractResult:
        """Run an extractor for a spec.

        **Phase 4 extension**: when ``extractor_id`` is provided explicitly
        OR when ``spec.extractors`` declares a ladder, the chosen binding
        drives behaviour. Per-binding ``options`` are merged into
        ``context["binding_options"]`` so extractors can read them.

        Selection order (first match wins):
          1. Explicit ``extractor_id`` kwarg — caller asked for a specific
             tier (e.g. user clicked "re-ingest with OCR").
          2. Ladder default: ``spec.default_binding()`` when
             ``spec.has_ladder()`` is True.
          3. Phase 1-3 fallback: ``spec.extractor`` as a synthetic binding.

        Enriches the returned ExtractResult's ``provenance`` dict with:
          - ``extractor_id``: which callable ran (lookup key)
          - ``extractor_version``: version of the handler
          - ``duration_ms``: wall-clock time to extract
          - ``type_name``: which type this file matched
          - ``cost_tier`` + ``fidelity``: binding metadata (Phase 4)

        Never raises on extractor error — returns an ExtractResult with
        ``provenance["error"]`` set and an empty elements list. The caller
        (kb/service.py) decides whether to fail the ingest or continue.
        """
        # Resolve the binding
        binding: ExtractorBinding | None = None
        if extractor_id:
            binding = spec.find_binding(extractor_id)
            if binding is None and extractor_id in self._extractors:
                # Caller asked for a registered extractor that isn't in the
                # spec's declared ladder (e.g. plugins, manual overrides).
                # Honour the request with a synthetic binding.
                binding = ExtractorBinding(extractor=extractor_id)
        if binding is None:
            binding = spec.default_binding()

        fn = self._extractors.get(binding.extractor)
        if not fn:
            raise FileTypeError(
                f"no extractor registered: '{binding.extractor}' "
                f"(type={spec.type_name}, file={path.name})"
            )

        started = time.perf_counter()
        ctx: ExtractorContext = dict(context or {})
        # Per-binding options take precedence over spec-level defaults and
        # are surfaced as a distinct context key so extractors can read
        # them without clobbering the caller's own context entries.
        if binding.options:
            merged = dict(spec.extractor_options or {})
            merged.update(binding.options)
            ctx["binding_options"] = merged
        elif spec.extractor_options:
            ctx["binding_options"] = dict(spec.extractor_options)
        try:
            result = await fn(path, spec, ctx)
        except Exception as exc:
            duration_ms = int((time.perf_counter() - started) * 1000)
            log.warning(
                "FileTypeRegistry: extractor '%s' raised on %s: %s",
                binding.extractor, path.name, exc,
            )
            return ExtractResult(
                elements=[],
                doc_metadata={},
                media=[],
                provenance={
                    "extractor_id": binding.extractor,
                    "extractor_version": _extractor_version(binding.extractor),
                    "duration_ms": duration_ms,
                    "type_name": spec.type_name,
                    "cost_tier": binding.cost_tier,
                    "fidelity": binding.fidelity,
                    "error": f"{type(exc).__name__}: {exc}",
                },
            )

        # Merge provenance — caller's values take precedence over defaults
        # so extractors can stamp their own `extractor_version` etc.
        duration_ms = int((time.perf_counter() - started) * 1000)
        prov: dict[str, Any] = {
            "extractor_id": binding.extractor,
            "extractor_version": _extractor_version(binding.extractor),
            "duration_ms": duration_ms,
            "type_name": spec.type_name,
            "cost_tier": binding.cost_tier,
            "fidelity": binding.fidelity,
        }
        prov.update(result.provenance or {})
        result.provenance = prov
        return result


# ── Version tracking for provenance ─────────────────────────────────────

# Per-extractor version strings — bumped when the extractor's behaviour
# changes in a way that warrants re-ingesting older content. Format:
#   "<name>@<semver-ish>"
# Consulted by `registry.extract()` and stamped on the ExtractResult so
# later re-ingest decisions ("is this content extracted with the current
# best handler?") become queryable against the graph.
_EXTRACTOR_VERSIONS: dict[str, str] = {  # arch:deterministic — extractor-version registry, not user-tunable strategy
    "text_utf8":       "text_utf8@1.0.0",
    # 1.2.0 — Phase 5: embedded image extraction + vision-LLM description
    # + parent_id linkage. Each embedded image becomes an Image element
    # with a MediaRef back-ref; optional sibling paragraph carries the
    # vision description linked via parent_id. Opt-in via
    # `extract_embedded_images` in extractor/binding options (default off
    # so cost stays predictable). When disabled, behaviour is byte-identical
    # to 1.1.0. The version bump lets downstream re-ingest proposals
    # distinguish "pre-multi-modal" (≤1.1.0) from "post" content so users
    # can opt into upgrading specific sources.
    #
    # 1.1.0 — Phase 2: Heading / Table / Paragraph elements via font-size
    # clustering + extract_tables(). Degrades to 1.0.0 per-page paragraphs
    # when char/font data is unavailable.
    "pdf_pdfplumber":  "pdf_pdfplumber@1.2.0",
    "vision_llm":      "vision_llm@1.0.0",
    "csv_tabular":     "csv_tabular@1.0.0",
    "json_pretty":     "json_pretty@1.0.0",
    "vtt_srt":         "vtt_srt@1.0.0",
    "html_parser":     "html_parser@1.0.0",
    # Phase 3 — new formats, element-aware from day one.
    "docx_python_docx":    "docx_python_docx@1.0.0",
    "epub_ebooklib":       "epub_ebooklib@1.0.0",
    "audio_whisper":       "audio_whisper@1.0.0",
    "video_ffmpeg_vision": "video_ffmpeg_vision@1.0.0",
    # Phase 4 — second-tier extractor: same pdfplumber pipeline as 1.1.0
    # but with page.to_image() + pytesseract OCR for pages where
    # extract_text() returns empty. Costs more (per-page OCR + 150dpi
    # rendering) so it's the `capable` tier of the paper_pdf ladder.
    #
    # Phase 5 bump: 1.0.0 → 1.1.0 when the OCR tier started honoring the
    # ``extract_embedded_images`` option alongside its non-OCR sibling.
    # Without the opt-in, behaviour is byte-identical to 1.0.0.
    "pdf_pdfplumber_ocr":  "pdf_pdfplumber_ocr@1.1.0",
}


def _extractor_version(extractor_id: str) -> str:
    return _EXTRACTOR_VERSIONS.get(extractor_id, f"{extractor_id}@unknown")


def register_extractor_version(extractor_id: str, version: str) -> None:
    """Plugins / Phase 3+ extractors register their version here so
    provenance triples carry the right lineage."""
    _EXTRACTOR_VERSIONS[extractor_id] = f"{extractor_id}@{version}"


# ── ExtractionSelector (Phase 4) ───────────────────────────────────────


_TIER_RANK: dict[str, int] = {"fast": 0, "capable": 1, "reasoning": 2}  # arch:deterministic — ModelRouter tier ordering
_FIDELITY_RANK: dict[str, int] = {"low": 0, "medium": 1, "high": 2}  # arch:deterministic — ExtractorBinding fidelity ordering


class ExtractionSelector:
    """Pick the best ExtractorBinding for a (spec, context) pair.

    Phase 4 infrastructure. Policy (first match wins):

    1. Explicit ``requested_id`` — caller names a specific extractor, e.g.
       the user clicked "re-ingest with OCR" in the wiki UI or an upgrade
       proposal is running a specific tier against a historical run.
    2. Ladder walk with budget filter — keep only bindings whose
       ``cost_tier`` is ≤ ``max_cost_tier`` AND whose ``fidelity`` is ≥
       ``min_fidelity``. Within survivors, prefer the HIGHEST fidelity
       (on tie, the one the ladder declared first — usually the lowest
       cost). This maximises quality within budget.
    3. Fallback to ``spec.default_binding()`` — Phase 1-3 compat when the
       spec has no ladder, or the budget filter zeroed out every binding.

    Plugins + tests construct the selector with custom rank overrides so
    a future deployment can re-order tiers without a code change.
    """

    def __init__(
        self,
        *,
        tier_rank: dict[str, int] | None = None,
        fidelity_rank: dict[str, int] | None = None,
    ) -> None:
        self._tier_rank = dict(tier_rank or _TIER_RANK)
        self._fidelity_rank = dict(fidelity_rank or _FIDELITY_RANK)

    def select(
        self,
        spec: FileTypeSpec,
        *,
        requested_id: str | None = None,
        max_cost_tier: CostTier = "reasoning",
        min_fidelity: Fidelity = "low",
    ) -> ExtractorBinding:
        """Return the binding to use for this extraction."""
        # 1. Explicit request wins.
        if requested_id:
            binding = spec.find_binding(requested_id)
            if binding is not None:
                return binding
            # Unknown id on a laddered spec → fall back to the ladder walk
            # below rather than the synthetic-binding path. Callers who
            # absolutely need an unknown id use the registry's
            # ``extract(extractor_id=...)`` escape hatch directly.

        # 2. No ladder → Phase 1-3 behaviour.
        if not spec.has_ladder():
            return spec.default_binding()

        max_cost = self._tier_rank.get(max_cost_tier, 2)
        min_fid = self._fidelity_rank.get(min_fidelity, 0)
        candidates = [
            b for b in spec.extractors
            if self._tier_rank.get(b.cost_tier, 0) <= max_cost
            and self._fidelity_rank.get(b.fidelity, 0) >= min_fid
        ]
        if not candidates:
            # Budget filter zeroed everything — return the cheapest entry
            # (first in the ladder by convention) so we still produce a
            # result instead of raising.
            return spec.extractors[0]

        # Prefer HIGHER fidelity. Ties preserve ladder order (Python's
        # sort is stable — originally `fast` entry stays before `capable`
        # at the same fidelity).
        candidates.sort(
            key=lambda b: self._fidelity_rank.get(b.fidelity, 0),
            reverse=True,
        )
        return candidates[0]

    def available_tiers(self, spec: FileTypeSpec) -> list[ExtractorBinding]:
        """Return all bindings declared for a spec, in ladder order.
        Used by the UI to let users pick a specific tier for re-ingest.
        Falls back to ``[spec.default_binding()]`` for Phase 1-3 specs.
        """
        if spec.has_ladder():
            return list(spec.extractors)
        return [spec.default_binding()]


# ── Upgrade proposals (Phase 4 infrastructure, integration in 4.5) ─────


@dataclass
class UpgradeProposal:
    """A recommendation to re-extract an existing source with a better tier.

    Produced by ``propose_extractor_upgrades`` — the pure-function layer of
    the upgrade surface. Full vault-walker + API endpoint integration lands
    in Phase 4.5 (walks ``wiki/sources/`` frontmatter, reads the recorded
    ``extractor_version``, surfaces a list of ``UpgradeProposal`` for user
    approval).

    Fields:
      - ``source_path``: where the original source summary lives (opaque
        identifier — the caller interprets it as a path, slug, whatever)
      - ``type_name``: which FileTypeSpec.type_name this source classified as
      - ``current_extractor``: the extractor id that ran at ingest time
      - ``proposed_extractor``: the higher-fidelity binding's extractor id
      - ``proposed_tier`` + ``proposed_fidelity``: metadata so the UI can
        explain the upgrade (e.g. "capable tier · high fidelity")
      - ``reason``: short human-readable description
    """

    source_path: str
    type_name: str
    current_extractor: str
    proposed_extractor: str
    proposed_tier: CostTier
    proposed_fidelity: Fidelity
    reason: str = ""


def propose_extractor_upgrades(
    sources: list[dict[str, Any]],
    registry: FileTypeRegistry,
    *,
    max_proposals: int = 500,
) -> list[UpgradeProposal]:
    """Scan a list of source records and return upgrade proposals.

    ``sources`` is a list of dicts with these keys:
      - ``path``: opaque source identifier (e.g. wiki page path)
      - ``type_name``: the FileTypeSpec.type_name recorded at ingest
      - ``extractor_id``: the extractor that ran (from provenance)

    A proposal is emitted when:
      1. The source's ``type_name`` has a ladder AND
      2. The source's ``extractor_id`` is NOT the highest-fidelity binding
         available in the current registry.

    Pure function — no I/O, no vault walker. The Phase 4.5 integration
    wraps this with the actual vault-reading glue.
    """
    proposals: list[UpgradeProposal] = []
    selector = ExtractionSelector()

    for record in sources:
        if len(proposals) >= max_proposals:
            break
        type_name = record.get("type_name") or ""
        current_id = record.get("extractor_id") or ""
        if not type_name or not current_id:
            continue

        spec = registry.get_spec(type_name)
        if spec is None or not spec.has_ladder():
            continue

        # What's the best tier available for this source given an unlimited
        # budget? That's the proposal target.
        best = selector.select(spec, max_cost_tier="reasoning")
        if best.extractor == current_id:
            continue

        # Only propose UPGRADES — higher fidelity than what's recorded.
        current_binding = spec.find_binding(current_id)
        current_fid = _FIDELITY_RANK.get(
            current_binding.fidelity if current_binding else "medium", 1,
        )
        best_fid = _FIDELITY_RANK.get(best.fidelity, 1)
        if best_fid <= current_fid:
            continue

        proposals.append(UpgradeProposal(
            source_path=str(record.get("path", "")),
            type_name=type_name,
            current_extractor=current_id,
            proposed_extractor=best.extractor,
            proposed_tier=best.cost_tier,
            proposed_fidelity=best.fidelity,
            reason=(best.description or f"Higher fidelity available: {best.extractor}"),
        ))

    return proposals
