"""Bundled file-type extractors and specs.

Phase 1 of the file-type registry (decisions.md §2026-04-18). Ports the 7
existing extraction paths in `kb/service.py` into the uniform registry
shape without changing user-visible behaviour.

Each extractor is defined as an async callable that takes
(path, spec, context) and returns an ExtractResult. They are registered
with the shared FileTypeRegistry at module import via `build_default_registry()`.

**Design rules:**
- Extractors never raise on content errors — they return an ExtractResult
  with `provenance["error"]` set and one `paragraph` element carrying a
  diagnostic string. This matches the existing behaviour in `kb/service.py`
  where an unreadable file becomes ``"[Binary or unreadable file: ...]"``.
- Heavy imports (pdfplumber, PyPDF2) live inside the extractor callable —
  boot stays fast and formats with heavy deps don't force install costs on
  users who never use them.
- Frontmatter parsing stays in the `text_utf8` extractor so downstream
  consumers get clean element text + structured `doc_metadata`.
- For Phase 1, richer formats (PDF with tables, DOCX headings, audio
  segments) emit a single minimal element set. Phase 2+ expands per type.
"""

from __future__ import annotations

import csv
import io
import json
import logging
import re
from pathlib import Path
from typing import Any

from .file_types import (
    Element,
    ExtractorBinding,
    ExtractResult,
    FileTypeRegistry,
    FileTypeSpec,
    MediaRef,
)

log = logging.getLogger(__name__)


# ── Shared helpers ──────────────────────────────────────────────────────


_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL)


def _split_frontmatter(raw: str) -> tuple[dict[str, Any], str]:
    """Lightweight frontmatter split — returns (metadata_dict, body).

    Uses python-frontmatter when available for full YAML parsing; falls
    back to naive line parsing otherwise so extractors don't hard-depend
    on the library. Matches the parsing behaviour of ``VaultReader.read``.
    """
    if not raw.startswith("---"):
        return {}, raw
    try:
        import frontmatter as _fm
        post = _fm.loads(raw)
        return dict(post.metadata or {}), post.content
    except Exception:
        m = _FRONTMATTER_RE.match(raw)
        if not m:
            return {}, raw
        meta: dict[str, Any] = {}
        for line in m.group(1).splitlines():
            if ":" in line:
                k, _, v = line.partition(":")
                meta[k.strip()] = v.strip().strip('"').strip("'")
        return meta, raw[m.end():]


def _error_result(
    extractor_id: str,
    type_name: str,
    message: str,
) -> ExtractResult:
    """Build an ExtractResult that carries a diagnostic paragraph and an
    `error` provenance marker. Mirrors the string-based error fallbacks
    in the pre-registry pipeline.
    """
    return ExtractResult(
        elements=[Element(kind="paragraph", text=message)],
        doc_metadata={},
        media=[],
        provenance={"error": message, "extractor_id": extractor_id, "type_name": type_name},
    )


# ── Extractor 1: text_utf8 ──────────────────────────────────────────────
# Serves: md, txt, html, rst, and anything else that's plain text.
# Frontmatter is split into doc_metadata; the body becomes one paragraph.

async def extract_text_utf8(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        raw = path.read_text(encoding="utf-8", errors="replace")
    except OSError as exc:
        return _error_result(
            "text_utf8", spec.type_name,
            f"[Could not read {path.name}: {exc}]",
        )

    meta, body = _split_frontmatter(raw)

    # Surface the common URL fields so the caller (kb/service.py) can
    # continue to pick up `source_url` from Obsidian Clipper, our scraper,
    # or generic sources without changing its downstream logic.
    for key in ("source_url", "source", "url"):
        if key in meta:
            meta.setdefault("raw_source_url_candidate", meta[key])

    elements: list[Element] = []
    stripped = body.strip()
    if stripped:
        elements.append(Element(kind="paragraph", text=stripped))

    return ExtractResult(
        elements=elements,
        doc_metadata=meta,
        media=[],
        provenance={},
    )


# ── Phase 5: embedded image extraction helpers ──────────────────────────
#
# When ``extract_embedded_images`` is enabled in extractor/binding options,
# both PDF extractors iterate ``page.images`` and emit one ``Image`` element
# per embedded image plus an optional sibling ``paragraph`` carrying the
# vision-LLM description — linked via ``parent_id`` so downstream consumers
# can rebuild the multi-modal join.
#
# Opt-in by default because a single vision call per image adds real cost
# (a 50-page PDF with 100 embedded figures = 100 LLM calls). Callers turn
# it on per-KB in the skill YAML or per-request via ``context``.
#
# Gracefully degrades at every boundary: missing LLM, broken PIL, pdfplumber
# crop failure, vision timeout — each short-circuits to skipping that
# specific image without raising. The caller still gets a valid
# ExtractResult and any other images succeed independently.

# Skip tiny images (icons, bullets, separators) by default.
_PDF_EMBED_MIN_DIM_DEFAULT = 100  # arch:deterministic — size threshold default

# Default vision prompt for embedded images. Users override via
# ``embedded_image_vision_prompt`` in context or binding_options.
_PDF_EMBED_DEFAULT_PROMPT = (  # arch:deterministic — fallback prompt string
    "This is an image embedded inside a PDF document. Describe what it "
    "shows: labels, charts, diagrams, figures, graphs, photos, formulas. "
    "Keep the description concise (2-4 sentences)."
)


def _read_pdf_embed_option(context: dict[str, Any], key: str, default: Any = None) -> Any:
    """Read a Phase 5 option from either the caller's ``context`` or the
    registry's merged ``binding_options`` dict. Caller context wins on
    collision so per-request overrides trump YAML defaults. Matches the
    dual-source pattern the registry establishes in ``extract()``.
    """
    if key in context:
        return context[key]
    bo = context.get("binding_options")
    if isinstance(bo, dict) and key in bo:
        return bo[key]
    return default


async def _describe_pdf_image_bytes(
    image_bytes: bytes,
    mime: str,
    *,
    llm: Any,
    timeout: int,
    prompt: str,
) -> str:
    """Call the vision LLM on in-memory image bytes. Returns the trimmed
    description text or ``""`` on any failure (no LLM, timeout, error).

    Never raises. The caller falls back to emitting just the ``Image``
    element (no description paragraph) when this returns empty.
    """
    import asyncio as _asyncio
    import base64 as _b64
    if llm is None:
        return ""
    try:
        b64 = _b64.b64encode(image_bytes).decode("ascii")
        response = await _asyncio.wait_for(
            llm.complete_vision(image_b64=b64, mime=mime, prompt=prompt),
            timeout=timeout,
        )
        return (getattr(response, "content", "") or "").strip()
    except TimeoutError:
        log.debug("PDF embedded-image vision call timed out after %ss", timeout)
        return ""
    except Exception:
        log.debug("PDF embedded-image vision call failed", exc_info=True)
        return ""


async def _pdf_extract_embedded_images(
    page: Any,
    page_idx: int,
    pdf_path_name: str,
    *,
    context: dict[str, Any],
) -> tuple[list[Element], list[MediaRef], int]:
    """Extract embedded images from one PDF page.

    Returns ``(elements, media, described_count)``:
      - ``elements`` — one ``Image`` element per image (always), plus an
        optional sibling ``paragraph`` with the vision description linked
        via ``parent_id=img_id`` (only when description is non-empty).
      - ``media`` — one ``MediaRef(kind="image")`` per Image element, with
        ``element_id`` back-referencing the Image for structural retrieval.
      - ``described_count`` — count of images where vision produced text.

    Returns ``([], [], 0)`` when:
      - ``extract_embedded_images`` is not truthy in context (default path),
      - the page exposes no ``images`` attribute (old pdfplumber builds),
      - the page's image list is empty,
      - any top-level exception occurs (defensive — ingest never fails).

    This is the **only** place in the ingest pipeline that calls the vision
    LLM per embedded image, so it owns the cost policy + fail-open contract.
    """
    if not _read_pdf_embed_option(context, "extract_embedded_images", False):
        return [], [], 0

    try:
        image_entries = list(getattr(page, "images", []) or [])
    except Exception:
        return [], [], 0
    if not image_entries:
        return [], [], 0

    min_w = int(_read_pdf_embed_option(
        context, "min_embedded_image_width", _PDF_EMBED_MIN_DIM_DEFAULT,
    ))
    min_h = int(_read_pdf_embed_option(
        context, "min_embedded_image_height", _PDF_EMBED_MIN_DIM_DEFAULT,
    ))
    describe = bool(_read_pdf_embed_option(context, "describe_embedded_images", True))
    llm = context.get("llm")
    if llm is None:
        describe = False
    vision_timeout = int(_read_pdf_embed_option(
        context, "embedded_image_vision_timeout", 60,
    ))
    vision_prompt = str(_read_pdf_embed_option(
        context, "embedded_image_vision_prompt", _PDF_EMBED_DEFAULT_PROMPT,
    ))

    new_elements: list[Element] = []
    new_media: list[MediaRef] = []
    described = 0

    for img_idx, img_info in enumerate(image_entries):
        if not isinstance(img_info, dict):
            continue
        try:
            width = float(img_info.get("width", 0) or 0)
            height = float(img_info.get("height", 0) or 0)
        except (TypeError, ValueError):
            width = height = 0.0
        if width and height and (width < min_w or height < min_h):
            # Icons / bullets — skip to keep the vision budget honest.
            continue

        img_id = f"page{page_idx}_img{img_idx}"
        # Semantic MediaRef — points at the parent PDF with a page+index
        # fragment rather than extracting bytes to disk. Downstream consumers
        # that need pixels re-open the PDF and crop the recorded bbox.
        semantic_path = f"{pdf_path_name}#page={page_idx}&img={img_idx}"

        try:
            bbox = (
                float(img_info.get("x0", 0) or 0),
                float(img_info.get("top", 0) or 0),
                float(img_info.get("x1", 0) or 0),
                float(img_info.get("bottom", 0) or 0),
            )
        except (TypeError, ValueError):
            bbox = (0.0, 0.0, 0.0, 0.0)

        description = ""
        if describe:
            png_bytes = b""
            try:
                cropped = page.crop(bbox).to_image(resolution=150)
                pil_image = getattr(cropped, "original", None)
                if pil_image is not None:
                    if getattr(pil_image, "mode", None) not in ("RGB", "L"):
                        pil_image = pil_image.convert("RGB")
                    import io as _io
                    buf = _io.BytesIO()
                    pil_image.save(buf, format="PNG")
                    png_bytes = buf.getvalue()
            except Exception:
                log.debug(
                    "PDF image crop failed on page %d image %d",
                    page_idx, img_idx, exc_info=True,
                )

            if png_bytes:
                description = await _describe_pdf_image_bytes(
                    png_bytes, "image/png",
                    llm=llm,
                    timeout=vision_timeout,
                    prompt=vision_prompt,
                )
                if description:
                    described += 1

        attrs: dict[str, Any] = {
            "path": semantic_path,
            "alt": f"Embedded image on page {page_idx}",
            "caption": description or None,
            "width": width or None,
            "height": height or None,
            "bbox": list(bbox),
        }
        bbox_tuple: tuple[int, int, int, int] | None = None
        if any(b > 0 for b in bbox):
            bbox_tuple = (
                int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3]),
            )

        new_elements.append(Element(
            kind="image",
            text=f"[Embedded image: page {page_idx}, image {img_idx}]",
            attrs=attrs,
            page=page_idx,
            bbox=bbox_tuple,
            element_id=img_id,
        ))
        if description:
            new_elements.append(Element(
                kind="paragraph",
                text=f"[Page {page_idx} · Image {img_idx}]\n{description}",
                page=page_idx,
                parent_id=img_id,
            ))

        new_media.append(MediaRef(
            path=semantic_path,
            kind="image",
            mime="application/pdf-embedded-image",
            size_bytes=None,
            element_id=img_id,
        ))

    return new_elements, new_media, described


# ── Extractor 2: pdf_pdfplumber (v1.1.0 — element-rich) ─────────────────
#
# Phase 2 upgrade: emits Heading / Table / Paragraph elements rather than
# one paragraph per page. Strategy:
#
#   1. Walk every page once, collecting `page.chars` to compute the document's
#      median font size (the "baseline"). Lines with avg size > baseline × 1.3
#      are flagged as headings; size ratio picks depth (1-4).
#   2. For each page, call `page.extract_tables()` — each table becomes one
#      `Element(kind="table")` with structured `attrs["rows"]` + `attrs["headers"]`.
#   3. Remaining body text is grouped by `page.chars[..., top]` into lines,
#      lines into paragraph blocks on blank-line boundaries.
#   4. If font analysis returns empty or collapses (e.g., PDF without char
#      metadata), fall back gracefully to the Phase 1 behavior: one paragraph
#      per page with a `[Page N]` prefix. Never emit zero elements when the
#      page had any extractable text.
#
# Downstream consumers that only need string output use `result.as_text()` —
# which joins element text with `\n\n`. Richer consumers (Phase 5 multi-modal
# join, compile, synthesize) read `elements` directly.

# Heading-detection thresholds. Tuned against common PDFs (A4 body = 10-12pt,
# h1 = 20-24pt, h2 = 16-18pt). Conservative: body wiggle below 1.3× baseline
# is NOT a heading to avoid false positives on slightly emphasised text.
_PDF_HEADING_RATIO = 1.3  # arch:deterministic — font-size clustering threshold
_PDF_HEADING_DEPTHS: tuple[tuple[float, int], ...] = (
    (2.0, 1),   # ≥2×    → h1
    (1.6, 2),   # ≥1.6×  → h2
    (1.4, 3),   # ≥1.4×  → h3
    (_PDF_HEADING_RATIO, 4),  # ≥1.3× → h4
)


def _pdf_heading_depth(ratio: float) -> int | None:
    """Return heading depth (1-4) for a size ratio, or None if below threshold."""
    if ratio < _PDF_HEADING_RATIO:
        return None
    for cutoff, depth in _PDF_HEADING_DEPTHS:
        if ratio >= cutoff:
            return depth
    return 4  # unreachable because the loop covers down to _PDF_HEADING_RATIO


def _pdf_median_size(pages: list) -> float:
    """Compute the document's baseline font size from page.chars metadata.

    Returns 0.0 when no usable size data is found — the caller treats that
    as the fallback signal and degrades to Phase 1 per-page paragraphs.
    """
    sizes: list[float] = []
    for page in pages:
        try:
            chars = getattr(page, "chars", None) or []
        except Exception:
            continue
        for ch in chars:
            size = ch.get("size") if isinstance(ch, dict) else None
            if isinstance(size, int | float) and size > 0:
                sizes.append(float(size))
    if not sizes:
        return 0.0
    sizes.sort()
    mid = len(sizes) // 2
    return sizes[mid] if len(sizes) % 2 else (sizes[mid - 1] + sizes[mid]) / 2.0


def _pdf_render_table(rows: list[list[str | None]]) -> str:
    """Human-readable table rendering for `Element.text` on table elements.
    Mirrors the CSV extractor's preview format so `as_text()` output stays
    consistent across structured-table types.
    """
    if not rows:
        return ""
    lines = [" | ".join((c or "") for c in rows[0])]
    if len(rows) > 1:
        lines.append(" | ".join(["---"] * len(rows[0])))
        for r in rows[1:20]:
            lines.append(" | ".join((c or "") for c in r))
        if len(rows) > 21:
            lines.append(f"... ({len(rows) - 21} more rows)")
    return "\n".join(lines)


def _pdf_extract_lines_with_size(page) -> list[tuple[float, str]]:
    """Group page.chars into lines (by rounded `top`), average font size
    per line. Returns [(avg_size, line_text), …] in reading order.

    Returns an empty list when char data is missing — the caller falls
    through to `page.extract_text()` and paragraph-only emission.
    """
    try:
        chars = getattr(page, "chars", None) or []
    except Exception:
        return []
    if not chars:
        return []

    # Bucket by rounded top coordinate (1pt bands ≈ one line height)
    by_top: dict[int, list[dict[str, Any]]] = {}
    for ch in chars:
        if not isinstance(ch, dict):
            continue
        top_raw = ch.get("top")
        if not isinstance(top_raw, int | float):
            continue
        by_top.setdefault(int(round(float(top_raw))), []).append(ch)

    lines: list[tuple[float, str]] = []
    # Top coordinate increases downward in pdfplumber — sort ascending for reading order
    for top in sorted(by_top.keys()):
        line_chars = sorted(
            by_top[top],
            key=lambda c: c.get("x0", 0) if isinstance(c.get("x0"), int | float) else 0,
        )
        text = "".join(
            (c.get("text", "") if isinstance(c.get("text"), str) else "")
            for c in line_chars
        ).strip()
        if not text:
            continue
        sizes = [
            float(c["size"]) for c in line_chars
            if isinstance(c.get("size"), int | float) and c["size"] > 0
        ]
        avg_size = sum(sizes) / len(sizes) if sizes else 0.0
        lines.append((avg_size, text))
    return lines


async def extract_pdf_pdfplumber(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    elements: list[Element] = []
    doc_meta: dict[str, Any] = {}

    # Primary: pdfplumber
    try:
        import pdfplumber
        with pdfplumber.open(path) as pdf:
            doc_meta["page_count"] = len(pdf.pages)
            # PDF-level metadata (title, author, etc.) — stripped leading "/"
            if pdf.metadata:
                for k, v in pdf.metadata.items():
                    if v and isinstance(v, str):
                        doc_meta[k.lower().strip("/")] = v

            baseline = _pdf_median_size(list(pdf.pages))

            embedded_media: list[MediaRef] = []
            embedded_image_count = 0
            embedded_described_count = 0

            for page_idx, page in enumerate(pdf.pages, start=1):
                # (a) Tables first — each becomes its own structured element.
                # Their text is later excluded from the paragraph stream via
                # line dedup below.
                table_texts: set[str] = set()
                try:
                    tables = page.extract_tables() or []
                except Exception:
                    tables = []
                for t_idx, table in enumerate(tables):
                    if not table:
                        continue
                    rows = [[(c or "") for c in row] for row in table]
                    # Heuristic: if first row looks like headers (all non-empty,
                    # each shorter than any data cell below), treat as headers.
                    headers = rows[0] if len(rows) > 1 and all(rows[0]) else None
                    elements.append(Element(
                        kind="table",
                        text=_pdf_render_table(rows),
                        attrs={"rows": rows[1:] if headers else rows, "headers": headers},
                        page=page_idx,
                        element_id=f"page{page_idx}_tbl{t_idx}",
                    ))
                    # Collect cell text so we can skip it in the paragraph pass
                    for row in rows:
                        for cell in row:
                            if cell and cell.strip():
                                table_texts.add(cell.strip())

                # (b) Font-size-driven heading + paragraph extraction.
                emitted_structured = False
                if baseline > 0:
                    lines = _pdf_extract_lines_with_size(page)
                    if lines:
                        para_buf: list[str] = []
                        for avg_size, line in lines:
                            # Skip text already captured by a table on this page
                            if line in table_texts:
                                continue
                            depth = (
                                _pdf_heading_depth(avg_size / baseline)
                                if avg_size > 0 else None
                            )
                            if depth is not None:
                                if para_buf:
                                    elements.append(Element(
                                        kind="paragraph",
                                        text="\n".join(para_buf).strip(),
                                        page=page_idx,
                                    ))
                                    para_buf = []
                                elements.append(Element(
                                    kind="heading",
                                    text=line,
                                    attrs={"depth": depth},
                                    page=page_idx,
                                ))
                                emitted_structured = True
                            else:
                                para_buf.append(line)
                        if para_buf:
                            elements.append(Element(
                                kind="paragraph",
                                text="\n".join(para_buf).strip(),
                                page=page_idx,
                            ))
                            emitted_structured = True

                # (c) Fallback: no structured output for this page → degrade
                # to Phase 1 behavior (one paragraph per page with [Page N] prefix).
                # This keeps downstream behavior identical on PDFs where font
                # analysis yields nothing useful (scanned-to-text with uniform
                # OCR output, for example).
                if not emitted_structured:
                    text = (page.extract_text() or "").strip()
                    if text:
                        elements.append(Element(
                            kind="paragraph",
                            text=f"[Page {page_idx}]\n{text}",
                            page=page_idx,
                        ))

                # (d) Phase 5: embedded image extraction + vision-LLM description.
                # Opt-in via `extract_embedded_images` in context or binding_options
                # (default off — vision calls per image add real cost). Helper
                # silently returns ([], [], 0) when disabled, so the default path
                # is identical to Phase 2 behaviour.
                img_elements, img_media, img_described = await _pdf_extract_embedded_images(
                    page, page_idx, path.name, context=context,
                )
                if img_elements:
                    elements.extend(img_elements)
                if img_media:
                    embedded_media.extend(img_media)
                    embedded_image_count += len(img_media)
                    embedded_described_count += img_described

        if elements:
            # Version-bump provenance — the extractor id stays "pdf_pdfplumber"
            # (registry-facing) but `extractor_version` records 1.2.0 so
            # re-ingest decisions can prefer richer extractions from now on.
            # 1.1.0 → 1.2.0 when Phase 5 embedded-image extraction + vision
            # description lands (opt-in, but the version bump lets downstream
            # re-ingest proposals distinguish "pre-multi-modal" from "post").
            provenance: dict[str, Any] = {
                "extractor": "pdfplumber",
                "pdf_extractor_version": "1.2.0",
            }
            if embedded_image_count:
                provenance["embedded_images_extracted"] = embedded_image_count
                provenance["embedded_images_described"] = embedded_described_count
            return ExtractResult(
                elements=elements,
                doc_metadata=doc_meta,
                media=embedded_media,
                provenance=provenance,
            )
        # Every page empty — likely scanned
        return ExtractResult(
            elements=[Element(
                kind="paragraph",
                text=f"[Scanned PDF: {path.name} — {doc_meta.get('page_count', 0)} pages, no extractable text]",
            )],
            doc_metadata=doc_meta,
            media=[],
            provenance={"extractor": "pdfplumber", "scanned": True},
        )
    except ImportError:
        log.debug("pdfplumber not installed; falling back to PyPDF2")
    except Exception as exc:
        log.debug("pdfplumber failed for %s: %s", path.name, exc)

    # Fallback: PyPDF2
    try:
        from PyPDF2 import PdfReader
        reader = PdfReader(str(path))
        doc_meta["page_count"] = len(reader.pages)
        for i, page in enumerate(reader.pages, start=1):
            text = (page.extract_text() or "").strip()
            if text:
                elements.append(Element(
                    kind="paragraph",
                    text=f"[Page {i}]\n{text}",
                    page=i,
                ))
        if elements:
            return ExtractResult(
                elements=elements,
                doc_metadata=doc_meta,
                media=[],
                provenance={"extractor": "PyPDF2"},
            )
        return ExtractResult(
            elements=[Element(
                kind="paragraph",
                text=f"[Scanned PDF: {path.name} — {doc_meta.get('page_count', 0)} pages, no extractable text]",
            )],
            doc_metadata=doc_meta,
            media=[],
            provenance={"extractor": "PyPDF2", "scanned": True},
        )
    except ImportError:
        return _error_result(
            "pdf_pdfplumber", spec.type_name,
            f"[PDF document: {path.name} — install pdfplumber for text extraction]",
        )
    except Exception as exc:
        return _error_result(
            "pdf_pdfplumber", spec.type_name,
            f"[PDF extraction failed: {path.name} — {exc}]",
        )


# ── Phase 4: pdf_pdfplumber_ocr@1.0.0 (capable tier) ────────────────────
#
# Second-tier PDF extractor. Same pdfplumber-based structured pipeline as
# ``extract_pdf_pdfplumber`` for pages with embedded text, BUT when a page
# has no extractable text (scanned/image PDFs), renders the page to an
# image via pdfplumber's ``page.to_image()`` and runs pytesseract on it.
# Emits a ``paragraph`` element tagged ``[Page N — OCR]`` so provenance is
# transparent.
#
# Gracefully falls back to the Phase 2 "scanned PDF sentinel" behaviour
# when pytesseract isn't installed OR OCR fails on a page.
#
# ExtractionSelector routes re-ingest requests here via the ladder
# (``paper_pdf`` spec declares this as the capable-tier binding).

def _pdf_ocr_page_image(page) -> str:
    """Render a pdfplumber page to a PIL image and OCR it via pytesseract.

    Returns the OCR'd text (stripped) or empty string when OCR is
    unavailable / fails. All imports are local so the `[ocr]` extra stays
    optional — missing pytesseract or Wand/pypdfium2 are degrade paths,
    not errors.

    We call ``pytesseract.image_to_string`` directly on the in-memory PIL
    image rather than routing through ``agntspace.llm.ocr.extract_text``
    (which takes a Path) to avoid a disk round-trip per page on long PDFs.
    """
    try:
        import pytesseract  # type: ignore[import-untyped]
    except ImportError:
        return ""
    # pdfplumber's to_image() wraps the page via pdfplumber.display, which
    # uses Wand or pypdfium2 under the hood. Some environments don't have
    # the native dep — treat render-failure the same as OCR-failure.
    try:
        page_img = page.to_image(resolution=150)
        pil_image = page_img.original  # pdfplumber.display.PageImage → PIL.Image
    except Exception:
        return ""
    try:
        # Ensure RGB/L mode — pytesseract rejects palette + RGBA on some
        # backends (matches the check in agntspace.llm.ocr.extract_text).
        if getattr(pil_image, "mode", None) not in ("RGB", "L"):
            pil_image = pil_image.convert("RGB")
        text = pytesseract.image_to_string(pil_image) or ""
        return text.strip()
    except Exception:
        return ""


async def extract_pdf_pdfplumber_ocr(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    """Capable-tier PDF extractor with page-level OCR fallback.

    Strategy per page:
      1. If pdfplumber finds extractable text → use Phase 2's font-cluster
         pipeline (Heading / Table / Paragraph elements, table dedup).
      2. If extraction returns empty AND page.to_image() + pytesseract
         produce text → emit a ``paragraph`` element with ``[Page N — OCR]``
         prefix so the OCR lineage is visible in provenance.
      3. If OCR also yields nothing → emit the Phase 2 scanned-PDF
         sentinel (still degraded vs. Phase 1's per-page paragraph, but
         consistent with the extractor id's "we tried OCR" contract).

    Provenance stamps `ocr_pages` (count of pages where OCR was used) so
    the wiki UI can surface "extracted with OCR fallback" badges.
    """
    elements: list[Element] = []
    doc_meta: dict[str, Any] = {}
    ocr_page_count = 0
    embedded_media: list[MediaRef] = []
    embedded_image_count = 0
    embedded_described_count = 0

    try:
        import pdfplumber
        with pdfplumber.open(path) as pdf:
            doc_meta["page_count"] = len(pdf.pages)
            if pdf.metadata:
                for k, v in pdf.metadata.items():
                    if v and isinstance(v, str):
                        doc_meta[k.lower().strip("/")] = v

            baseline = _pdf_median_size(list(pdf.pages))

            for page_idx, page in enumerate(pdf.pages, start=1):
                # (a) Tables first (same as Phase 2)
                table_texts: set[str] = set()
                try:
                    tables = page.extract_tables() or []
                except Exception:
                    tables = []
                for t_idx, table in enumerate(tables):
                    if not table:
                        continue
                    rows = [[(c or "") for c in row] for row in table]
                    headers = rows[0] if len(rows) > 1 and all(rows[0]) else None
                    elements.append(Element(
                        kind="table",
                        text=_pdf_render_table(rows),
                        attrs={
                            "rows": rows[1:] if headers else rows,
                            "headers": headers,
                        },
                        page=page_idx,
                        element_id=f"page{page_idx}_tbl{t_idx}",
                    ))
                    for row in rows:
                        for cell in row:
                            if cell and cell.strip():
                                table_texts.add(cell.strip())

                # (b) Font-cluster structured extraction
                emitted_structured = False
                if baseline > 0:
                    lines = _pdf_extract_lines_with_size(page)
                    if lines:
                        para_buf: list[str] = []
                        for avg_size, line in lines:
                            if line in table_texts:
                                continue
                            depth = (
                                _pdf_heading_depth(avg_size / baseline)
                                if avg_size > 0 else None
                            )
                            if depth is not None:
                                if para_buf:
                                    elements.append(Element(
                                        kind="paragraph",
                                        text="\n".join(para_buf).strip(),
                                        page=page_idx,
                                    ))
                                    para_buf = []
                                elements.append(Element(
                                    kind="heading",
                                    text=line,
                                    attrs={"depth": depth},
                                    page=page_idx,
                                ))
                                emitted_structured = True
                            else:
                                para_buf.append(line)
                        if para_buf:
                            elements.append(Element(
                                kind="paragraph",
                                text="\n".join(para_buf).strip(),
                                page=page_idx,
                            ))
                            emitted_structured = True

                # (c) Page-level plain text fallback
                if not emitted_structured:
                    text = (page.extract_text() or "").strip()
                    if text:
                        elements.append(Element(
                            kind="paragraph",
                            text=f"[Page {page_idx}]\n{text}",
                            page=page_idx,
                        ))
                        emitted_structured = True

                # (d) OCR fallback — THE Phase-4 differentiator.
                # Only run when the page produced zero text above.
                if not emitted_structured:
                    ocr_text = _pdf_ocr_page_image(page)
                    if ocr_text:
                        ocr_page_count += 1
                        elements.append(Element(
                            kind="paragraph",
                            text=f"[Page {page_idx} — OCR]\n{ocr_text}",
                            page=page_idx,
                            attrs={"ocr": True},
                        ))

                # (e) Phase 5 — embedded image extraction + vision LLM
                # description, gated on the ``extract_embedded_images`` opt-in
                # flag in context/binding_options. Shared helper is the same
                # one ``extract_pdf_pdfplumber`` uses so both tiers behave
                # identically under the option. Silent no-op when disabled,
                # so the Phase 4 OCR default path is byte-identical.
                img_elements, img_media, img_described = await _pdf_extract_embedded_images(
                    page, page_idx, path.name, context=context,
                )
                if img_elements:
                    elements.extend(img_elements)
                if img_media:
                    embedded_media.extend(img_media)
                    embedded_image_count += len(img_media)
                    embedded_described_count += img_described

        if elements:
            provenance: dict[str, Any] = {
                "extractor": "pdfplumber+ocr",
                # Phase 5 bump: ocr-1.0.0 → ocr-1.1.0 when the OCR extractor
                # started honoring the same embedded-image-extraction option
                # as the non-OCR tier. Behaviour without the opt-in is
                # byte-identical to ocr-1.0.0.
                "pdf_extractor_version": "ocr-1.1.0",
                "ocr_pages": ocr_page_count,
            }
            if embedded_image_count:
                provenance["embedded_images_extracted"] = embedded_image_count
                provenance["embedded_images_described"] = embedded_described_count
            return ExtractResult(
                elements=elements,
                doc_metadata=doc_meta,
                media=embedded_media,
                provenance=provenance,
            )
        # Every page empty even after OCR — honest sentinel
        return ExtractResult(
            elements=[Element(
                kind="paragraph",
                text=f"[Scanned PDF: {path.name} — {doc_meta.get('page_count', 0)} pages, no text recovered via OCR]",
            )],
            doc_metadata=doc_meta,
            media=[],
            provenance={
                "extractor": "pdfplumber+ocr",
                "pdf_extractor_version": "ocr-1.1.0",
                "scanned": True,
                "ocr_pages": 0,
            },
        )
    except ImportError:
        return _error_result(
            "pdf_pdfplumber_ocr", spec.type_name,
            f"[PDF document: {path.name} — install pdfplumber for text extraction]",
        )
    except Exception as exc:
        return _error_result(
            "pdf_pdfplumber_ocr", spec.type_name,
            f"[PDF OCR extraction failed: {path.name} — {exc}]",
        )


# ── Extractor 3: vision_llm ─────────────────────────────────────────────
# Async — calls the OCR pipeline + multimodal LLM. Requires `llm` in
# context (the caller's LLM provider). Emits an Image element (with
# MediaRef back-ref), plus paragraphs for OCR text and vision description.

async def extract_vision_llm(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    import asyncio

    llm = context.get("llm")
    describe_image = context.get("describe_image")  # optional service method
    if llm is None and describe_image is None:
        return _error_result(
            "vision_llm", spec.type_name,
            f"[Image file: {path.name} — vision unavailable (no LLM wired)]",
        )

    # OCR pass — optional, skip if Tesseract not installed
    ocr_text = ""
    try:
        from agntspace.llm.ocr import extract_text as ocr_extract
        ocr_text = ocr_extract(path) or ""
    except Exception:
        log.debug("OCR failed for %s", path.name, exc_info=True)

    # Vision LLM pass
    description = ""
    try:
        if describe_image is not None:
            description = await asyncio.wait_for(
                describe_image(path, ocr_text=ocr_text),
                timeout=context.get("vision_timeout", 120),
            )
        elif llm is not None:
            # Minimal fallback — call the provider's vision endpoint directly
            raw_bytes = path.read_bytes()
            import base64
            b64 = base64.standard_b64encode(raw_bytes).decode("ascii")
            mime = f"image/{path.suffix.lstrip('.').lower().replace('jpg', 'jpeg')}"
            prompt = (
                "Describe what the image shows, any text/labels visible, and context. "
                f"OCR captured this text:\n{ocr_text}" if ocr_text else
                "Describe what the image shows, any text/labels visible, and context."
            )
            response = await asyncio.wait_for(
                llm.complete_vision(image_b64=b64, mime=mime, prompt=prompt),
                timeout=context.get("vision_timeout", 120),
            )
            description = getattr(response, "content", "") or ""
    except TimeoutError:
        description = f"[Vision timed out for {path.name}]"
    except Exception as exc:
        log.debug("Vision LLM failed for %s", path.name, exc_info=True)
        description = f"[Vision unavailable: {exc}]"

    # Build elements — Image first, then OCR text, then description
    img_id = f"img_{path.stem}"
    elements: list[Element] = [Element(
        kind="image",
        text=f"[Image: {path.name}]",
        attrs={"path": str(path), "alt": path.stem, "caption": description or None},
        element_id=img_id,
    )]
    if ocr_text.strip():
        elements.append(Element(
            kind="paragraph",
            text=f"## OCR Text\n{ocr_text.strip()}",
            parent_id=img_id,
        ))
    if description.strip():
        elements.append(Element(
            kind="paragraph",
            text=f"## Visual Description\n{description.strip()}",
            parent_id=img_id,
        ))

    try:
        size = path.stat().st_size
    except OSError:
        size = None
    media = [MediaRef(
        path=str(path),
        kind="image",
        mime=f"image/{path.suffix.lstrip('.').lower()}",
        size_bytes=size,
        element_id=img_id,
    )]

    return ExtractResult(
        elements=elements,
        doc_metadata={"has_ocr": bool(ocr_text.strip()), "has_description": bool(description.strip())},
        media=media,
        provenance={"ocr_chars": len(ocr_text), "description_chars": len(description)},
    )


# ── Extractor 4: csv_tabular ────────────────────────────────────────────
# Emits one `table` element carrying rows + headers. For `as_text()`,
# the text field is a human-readable rendering so legacy consumers keep
# working.

async def extract_csv_tabular(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        raw = path.read_text(encoding="utf-8", errors="replace")
    except OSError as exc:
        return _error_result("csv_tabular", spec.type_name,
                             f"[Could not read {path.name}: {exc}]")

    rows: list[list[str]] = []
    try:
        reader = csv.reader(io.StringIO(raw))
        for row in reader:
            rows.append(row)
    except csv.Error as exc:
        return _error_result("csv_tabular", spec.type_name,
                             f"[CSV parse error in {path.name}: {exc}]")

    if not rows:
        return ExtractResult(
            elements=[Element(kind="paragraph", text=f"[Empty CSV: {path.name}]")],
            doc_metadata={"row_count": 0, "column_count": 0},
        )

    headers = rows[0] if len(rows) > 1 else None
    data_rows = rows[1:] if headers else rows

    # Human-readable text rendering for legacy consumers — format as a
    # markdown-ish table preview capped at 20 rows.
    preview_lines: list[str] = []
    if headers:
        preview_lines.append(" | ".join(headers))
        preview_lines.append(" | ".join(["---"] * len(headers)))
    for row in data_rows[:20]:
        preview_lines.append(" | ".join(row))
    if len(data_rows) > 20:
        preview_lines.append(f"... ({len(data_rows) - 20} more rows)")

    table_element = Element(
        kind="table",
        text="\n".join(preview_lines),
        attrs={"rows": data_rows, "headers": headers},
    )

    return ExtractResult(
        elements=[table_element],
        doc_metadata={
            "row_count": len(data_rows),
            "column_count": len(headers) if headers else (len(rows[0]) if rows else 0),
        },
        media=[],
        provenance={},
    )


# ── Extractor 5: json_pretty ────────────────────────────────────────────
# Handles JSON + JSONL. Dispatches to the existing parsers.py helpers for
# well-known shapes (Claude Code history, ChatGPT export, Slack export)
# and falls back to pretty-printed JSON for anything else.

async def extract_json_pretty(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        from agntspace.kb.parsers import parse_file
        parsed_text = parse_file(path)
    except Exception as exc:
        log.debug("parse_file failed for %s: %s", path.name, exc)
        parsed_text = None

    if parsed_text is None:
        try:
            raw = path.read_text(encoding="utf-8", errors="replace")
            # Try to pretty-print
            data = json.loads(raw) if path.suffix.lower() == ".json" else raw
            parsed_text = json.dumps(data, indent=2) if not isinstance(data, str) else data
        except Exception as exc:
            return _error_result("json_pretty", spec.type_name,
                                 f"[Could not parse {path.name}: {exc}]")

    return ExtractResult(
        elements=[Element(kind="paragraph", text=parsed_text)],
        doc_metadata={},
        media=[],
        provenance={},
    )


# ── Extractor 6: vtt_srt ────────────────────────────────────────────────
# Subtitles. Phase 1 emits one paragraph with the parsed transcript.
# Phase 3 will emit `captioned_media` elements per cue with start/end ms.

async def extract_vtt_srt(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        from agntspace.kb.parsers import parse_file
        text = parse_file(path)
    except Exception as exc:
        return _error_result("vtt_srt", spec.type_name,
                             f"[Transcript parse error in {path.name}: {exc}]")

    if not text:
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
        except OSError as exc:
            return _error_result("vtt_srt", spec.type_name,
                                 f"[Could not read {path.name}: {exc}]")

    return ExtractResult(
        elements=[Element(kind="paragraph", text=text or "[Empty transcript]")],
        doc_metadata={},
        media=[],
        provenance={},
    )


# ── Extractor 8: docx_python_docx ───────────────────────────────────────
#
# Phase 3 deliverable. Word's internal structure maps 1:1 to our Element
# kinds: `Heading 1-9` styles → `heading` with depth, bulleted/numbered list
# styles → `list_item`, tables → `table` with rows + optional headers,
# normal paragraphs → `paragraph`. Document-order is preserved by walking
# the body's OXML children in sequence (paragraphs + tables interleave in
# real documents; iterating `doc.paragraphs` and `doc.tables` separately
# would lose that ordering).
#
# Heavy import gated inside the callable — installs only when `[documents]`
# extra is pulled. Graceful error message otherwise.

_DOCX_HEADING_DEPTH_CAP = 6  # HTML goes h1-h6; clamp Heading 7-9 → 6


def _docx_heading_depth(style_name: str) -> int | None:
    """Return heading depth (1-6) for a Word style name, or None if not a heading.

    Matches ``Heading 1``–``Heading 9`` (case-insensitive). ``Title`` maps
    to depth 1. Any other style (including ``Subtitle``, ``Quote``, etc.)
    returns None so the caller emits a regular paragraph.
    """
    name = (style_name or "").strip()
    if not name:
        return None
    low = name.lower()
    if low == "title":
        return 1
    if low.startswith("heading"):
        # Extract trailing digit if present ("Heading 3" → 3)
        parts = low.split()
        if len(parts) == 2 and parts[1].isdigit():
            return min(_DOCX_HEADING_DEPTH_CAP, max(1, int(parts[1])))
        return 2  # bare "Heading" → default mid-level
    return None


def _docx_is_list_style(style_name: str) -> tuple[bool, bool]:
    """Inspect a Word style name for list semantics.

    Returns ``(is_list, ordered)``. Word conventions: ``List Bullet`` /
    ``List Bullet 2`` / ``List Paragraph`` are unordered; ``List Number`` /
    ``List Number 2`` are ordered.
    """
    name = (style_name or "").strip().lower()
    if "list" not in name:
        return False, False
    ordered = "number" in name
    return True, ordered


def _docx_list_depth(paragraph: Any) -> int:
    """Estimate list nesting depth from a paragraph's indentation.

    1 EMU = 914400 per inch; standard Word list indents are ~0.25" = 228600 EMU
    per level. Returns 1 when no indentation info is available.
    """
    try:
        fmt = paragraph.paragraph_format
        indent = fmt.left_indent
        if indent is None:
            return 1
        emu = int(indent)
        if emu <= 0:
            return 1
        return max(1, 1 + emu // 228600)
    except Exception:
        return 1


def _docx_render_table_rows(table: Any) -> list[list[str]]:
    """Extract table cells as a list of string rows, stripping whitespace."""
    rows: list[list[str]] = []
    try:
        for row in table.rows:
            rows.append([
                (cell.text or "").strip() for cell in row.cells
            ])
    except Exception:
        pass
    return rows


async def extract_docx_python_docx(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        from docx import Document
        from docx.oxml.ns import qn  # noqa: F401 — confirms the install is complete
        from docx.oxml.table import CT_Tbl
        from docx.oxml.text.paragraph import CT_P
        from docx.table import Table as DocxTable
        from docx.text.paragraph import Paragraph as DocxParagraph
    except ImportError:
        return _error_result(
            "docx_python_docx", spec.type_name,
            f"[DOCX document: {path.name} — install 'pip install agntspace[documents]' for text extraction]",
        )

    try:
        doc = Document(str(path))
    except Exception as exc:
        return _error_result(
            "docx_python_docx", spec.type_name,
            f"[Could not parse DOCX {path.name}: {exc}]",
        )

    doc_meta: dict[str, Any] = {}
    try:
        core = doc.core_properties
        for attr, key in (
            ("title", "title"),
            ("author", "author"),
            ("language", "language"),
            ("subject", "subject"),
            ("keywords", "keywords"),
            ("comments", "comments"),
        ):
            value = getattr(core, attr, None)
            if value:
                doc_meta[key] = str(value)
        if core.created:
            doc_meta["created"] = core.created.isoformat()
        if core.modified:
            doc_meta["modified"] = core.modified.isoformat()
    except Exception:
        pass

    elements: list[Element] = []
    # Walk the document body in OXML order. `iterchildren()` yields the
    # paragraphs (CT_P) and tables (CT_Tbl) in the exact order they appear
    # in the file — which is what we need to preserve reading flow.
    try:
        body_children = list(doc.element.body.iterchildren())
    except Exception as exc:
        log.debug("docx body iteration failed for %s: %s", path.name, exc)
        body_children = []

    for child in body_children:
        if isinstance(child, CT_P):
            try:
                para = DocxParagraph(child, doc)
            except Exception:
                continue
            text = (para.text or "").strip()
            if not text:
                continue
            style_name = getattr(para.style, "name", "") if para.style else ""
            depth = _docx_heading_depth(style_name)
            if depth is not None:
                elements.append(Element(
                    kind="heading", text=text, attrs={"depth": depth},
                ))
                continue
            is_list, ordered = _docx_is_list_style(style_name)
            if is_list:
                elements.append(Element(
                    kind="list_item",
                    text=text,
                    attrs={
                        "depth": _docx_list_depth(para),
                        "ordered": ordered,
                    },
                ))
                continue
            elements.append(Element(kind="paragraph", text=text))
        elif isinstance(child, CT_Tbl):
            try:
                tbl = DocxTable(child, doc)
            except Exception:
                continue
            rows = _docx_render_table_rows(tbl)
            if not rows:
                continue
            headers = rows[0] if len(rows) > 1 and all(rows[0]) else None
            elements.append(Element(
                kind="table",
                text=_pdf_render_table(rows),
                attrs={
                    "rows": rows[1:] if headers else rows,
                    "headers": headers,
                },
            ))

    # Degraded fallback: if OXML iteration produced nothing (uncommon, but
    # possible on malformed .docx), flatten paragraphs the simple way so
    # downstream LLM prompts still get text to chew on.
    if not elements:
        try:
            fallback = "\n\n".join(
                p.text.strip() for p in doc.paragraphs if (p.text or "").strip()
            )
        except Exception:
            fallback = ""
        if fallback:
            elements.append(Element(kind="paragraph", text=fallback))
        else:
            return _error_result(
                "docx_python_docx", spec.type_name,
                f"[DOCX {path.name} has no extractable text]",
            )

    return ExtractResult(
        elements=elements,
        doc_metadata=doc_meta,
        media=[],
        provenance={"docx_extractor_version": "1.0.0"},
    )


# ── Extractor 9: epub_ebooklib ──────────────────────────────────────────
#
# Phase 3 deliverable. ePubs are zipped HTML + metadata (OPF). Each chapter
# is an HTML document listed in the spine. Strategy:
#   1. Extract dc:title / dc:creator / dc:language into doc_metadata.
#   2. Walk spine items. For each, emit one `heading` (depth=1) with the
#      item's title (from the nav doc if available, else the filename), then
#      parse the HTML body via the same readable-text extractor `html_parser`
#      uses — splitting on blank lines into `paragraph` elements.
#   3. Fallback: if ebooklib can't read the file, return an error result.

async def extract_epub_ebooklib(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        from ebooklib import ITEM_DOCUMENT, epub  # type: ignore
    except ImportError:
        return _error_result(
            "epub_ebooklib", spec.type_name,
            f"[ePub document: {path.name} — install 'pip install agntspace[documents]' for text extraction]",
        )

    try:
        book = epub.read_epub(str(path))
    except Exception as exc:
        return _error_result(
            "epub_ebooklib", spec.type_name,
            f"[Could not parse ePub {path.name}: {exc}]",
        )

    doc_meta: dict[str, Any] = {}
    try:
        for dc_key, meta_key in (
            ("title", "title"),
            ("creator", "author"),
            ("language", "language"),
            ("publisher", "publisher"),
            ("description", "description"),
            ("identifier", "identifier"),
        ):
            values = book.get_metadata("DC", dc_key)
            if values:
                # Each entry is typically (value, {attr: val}); keep the value.
                doc_meta[meta_key] = str(values[0][0]) if values[0] else ""
    except Exception:
        pass

    # Build a file_name → nav_title map from the EPUB's nav doc when present,
    # so chapter headings carry real titles instead of bare filenames.
    title_by_href: dict[str, str] = {}
    try:
        for item in book.get_items():
            content = item.get_content() if hasattr(item, "get_content") else b""
            if b"<nav" not in content:
                continue
            try:
                # Very lightweight title extraction — nav docs are HTML with
                # anchor elements; we pull href + link text pairs.
                text = content.decode("utf-8", errors="replace")
                import re as _re
                for m in _re.finditer(
                    r'<a\s+[^>]*href="([^"#]+)(?:#[^"]*)?"[^>]*>([^<]+)</a>',
                    text,
                ):
                    href, link_text = m.group(1), m.group(2).strip()
                    if href and link_text:
                        title_by_href.setdefault(href, link_text)
            except Exception:
                continue
    except Exception:
        pass

    elements: list[Element] = []
    try:
        from agntspace.integrations.browser import extract_readable_text as _html_text
    except Exception:
        _html_text = None  # type: ignore

    # Iterate in spine order so elements match reading flow, not archive order.
    try:
        for (ref, _linear) in book.spine:
            item = book.get_item_with_id(ref)
            if item is None or getattr(item, "media_type", None) != "application/xhtml+xml":
                continue
            if getattr(item, "get_type", lambda: None)() not in (ITEM_DOCUMENT, None):
                # Some ebooklib versions return ITEM_DOCUMENT from get_type
                pass

            raw = item.get_content().decode("utf-8", errors="replace")
            # Chapter title — prefer nav doc mapping, else HTML <title>, else file_name
            chapter_title = (
                title_by_href.get(item.file_name, "")
                or title_by_href.get(item.file_name.rsplit("/", 1)[-1], "")
            )
            if not chapter_title:
                import re as _re
                m = _re.search(r"<title[^>]*>([^<]+)</title>", raw, _re.IGNORECASE)
                if m:
                    chapter_title = m.group(1).strip()
            if not chapter_title:
                chapter_title = item.file_name

            elements.append(Element(
                kind="heading",
                text=chapter_title,
                attrs={"depth": 1},
            ))

            # Body text — reuse the browser's readable-text extractor so
            # the same stripping rules apply that html_parser uses.
            body_text = ""
            if _html_text is not None:
                try:
                    body_text = _html_text(raw) or ""
                except Exception:
                    body_text = ""
            if not body_text:
                # Conservative stdlib fallback
                import re as _re
                body_text = _re.sub(r"<[^>]+>", " ", raw)
                body_text = _re.sub(r"\s+", " ", body_text).strip()

            # Split on blank-line boundaries into paragraph elements
            for para_raw in body_text.split("\n\n"):
                para = para_raw.strip()
                if para:
                    elements.append(Element(kind="paragraph", text=para))
    except Exception as exc:
        log.debug("ePub spine walk failed for %s: %s", path.name, exc)

    if not elements:
        return _error_result(
            "epub_ebooklib", spec.type_name,
            f"[ePub {path.name} has no extractable text]",
        )

    return ExtractResult(
        elements=elements,
        doc_metadata=doc_meta,
        media=[],
        provenance={"epub_extractor_version": "1.0.0"},
    )


# ── Extractor 10: audio_whisper ─────────────────────────────────────────
#
# Phase 3 deliverable. Routes through the existing ``agntspace.llm.transcribe``
# multi-backend pipeline (OpenAI Whisper API → LiteLLM → faster-whisper →
# openai-whisper, first available wins). Returns the transcript as a single
# `paragraph` element. Phase 3.5+ can upgrade to `captioned_media` elements
# with start_ms/end_ms when faster-whisper's segment timestamps are exposed.

async def extract_audio_whisper(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        from agntspace.llm.transcribe import (
            NO_BACKEND_AVAILABLE,
            transcribe_audio,
        )
    except ImportError:
        return _error_result(
            "audio_whisper", spec.type_name,
            f"[Audio file: {path.name} — transcription module unavailable]",
        )

    # transcribe_audio is async in the existing module; keep a conservative
    # cap so one stuck transcription can't block the ingest batch.
    import asyncio as _asyncio
    timeout = context.get("transcribe_timeout", 180)
    try:
        transcript = await _asyncio.wait_for(
            transcribe_audio(path), timeout=timeout,
        )
    except TimeoutError:
        return _error_result(
            "audio_whisper", spec.type_name,
            f"[Audio transcription timed out after {timeout}s: {path.name}]",
        )
    except Exception as exc:
        return _error_result(
            "audio_whisper", spec.type_name,
            f"[Audio transcription failed for {path.name}: {exc}]",
        )

    transcript = (transcript or "").strip()
    if not transcript or transcript == NO_BACKEND_AVAILABLE:
        return _error_result(
            "audio_whisper", spec.type_name,
            transcript or f"[Audio file: {path.name} — no transcription backend installed]",
        )

    try:
        size = path.stat().st_size
    except OSError:
        size = None

    return ExtractResult(
        elements=[Element(kind="paragraph", text=transcript)],
        doc_metadata={"transcript_chars": len(transcript)},
        media=[MediaRef(path=str(path), kind="audio", size_bytes=size)],
        provenance={"audio_extractor_version": "1.0.0"},
    )


# ── Extractor 11: video_ffmpeg_vision ───────────────────────────────────
#
# Phase 3 deliverable. Reuses the ffmpeg subprocess pattern from
# ``channels/whatsapp.py`` but produces registry-native Elements:
#   - Image element + MediaRef for the extracted keyframe
#   - Paragraph element for the vision LLM's description of the keyframe
#   - Paragraph element for the audio transcript (if transcription succeeds)
#
# Falls back to a metadata-only description when ffmpeg isn't installed.

async def extract_video_ffmpeg_vision(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    import asyncio as _asyncio
    import subprocess

    llm = context.get("llm")
    vision_timeout = context.get("video_vision_timeout", 60)
    transcribe_timeout = context.get("video_transcribe_timeout", 180)

    try:
        size = path.stat().st_size
    except OSError:
        size = None

    elements: list[Element] = []
    media: list[MediaRef] = [MediaRef(path=str(path), kind="video", size_bytes=size)]
    doc_meta: dict[str, Any] = {}
    frame_description = ""

    # 1. Keyframe at 00:00:01 via ffmpeg — skip if binary missing.
    frame_path: Path | None = None
    try:
        import tempfile
        tmp_dir = tempfile.mkdtemp(prefix="agnt-video-frame-")
        frame_path_candidate = Path(tmp_dir) / f"{path.stem}.jpg"
        result = subprocess.run(
            [
                "ffmpeg", "-i", str(path), "-ss", "00:00:01",
                "-frames:v", "1", "-y", str(frame_path_candidate),
            ],
            capture_output=True, timeout=20,
        )
        if result.returncode == 0 and frame_path_candidate.is_file():
            frame_path = frame_path_candidate
    except FileNotFoundError:
        log.debug("ffmpeg not installed — video frame extraction skipped")
    except (subprocess.TimeoutExpired, Exception):
        log.debug("ffmpeg frame extraction failed for %s", path.name, exc_info=True)

    # 2. Vision LLM on the keyframe (when one was extracted + llm available)
    if frame_path and llm is not None:
        try:
            import base64 as _b64
            raw_bytes = frame_path.read_bytes()
            b64 = _b64.b64encode(raw_bytes).decode("ascii")
            mime = "image/jpeg"
            prompt = "This is a frame from a video. Describe what it shows."
            response = await _asyncio.wait_for(
                llm.complete_vision(image_b64=b64, mime=mime, prompt=prompt),
                timeout=vision_timeout,
            )
            frame_description = (getattr(response, "content", "") or "").strip()
        except TimeoutError:
            frame_description = f"[Vision timed out on frame from {path.name}]"
        except Exception as exc:
            log.debug("Vision LLM failed on video frame: %s", exc, exc_info=True)

    img_id = f"video_{path.stem}_frame1"
    if frame_path:
        elements.append(Element(
            kind="image",
            text=f"[Video frame 1s: {path.name}]",
            attrs={"path": str(frame_path), "caption": frame_description or None},
            element_id=img_id,
        ))
        media.append(MediaRef(
            path=str(frame_path), kind="image", mime="image/jpeg",
            element_id=img_id,
        ))
    if frame_description:
        elements.append(Element(
            kind="paragraph",
            text=f"## Frame Description\n{frame_description}",
            parent_id=img_id if frame_path else None,
        ))

    # 3. Audio transcript — reuse the audio extractor's contract.
    try:
        from agntspace.llm.transcribe import (
            NO_BACKEND_AVAILABLE,
            transcribe_audio,
        )
        transcript = await _asyncio.wait_for(
            transcribe_audio(path), timeout=transcribe_timeout,
        )
        transcript = (transcript or "").strip()
        if transcript and transcript != NO_BACKEND_AVAILABLE:
            elements.append(Element(
                kind="paragraph",
                text=f"## Audio Transcript\n{transcript}",
            ))
            doc_meta["transcript_chars"] = len(transcript)
    except TimeoutError:
        log.debug("Video audio transcription timed out for %s", path.name)
    except Exception as exc:
        log.debug("Video audio transcription failed for %s: %s", path.name, exc)

    if not elements:
        # Metadata-only fallback — mirrors whatsapp.py's fallback string.
        size_kb = int(size / 1024) if size else 0
        elements.append(Element(
            kind="paragraph",
            text=(
                f"[Video file: {path.name} ({size_kb} KB). No frame extraction "
                f"available — install ffmpeg for visual analysis + faster-whisper "
                f"for audio transcription.]"
            ),
        ))

    return ExtractResult(
        elements=elements,
        doc_metadata=doc_meta,
        media=media,
        provenance={"video_extractor_version": "1.0.0"},
    )


# ── Extractor 7: html_parser ────────────────────────────────────────────
# Strips tags + scripts + styles, keeps readable text. The caller applies
# `_strip_web_noise` on top for KB-specific cleanup — that's a consumer
# policy, not an extractor concern.

async def extract_html_parser(
    path: Path,
    spec: FileTypeSpec,
    context: dict[str, Any],
) -> ExtractResult:
    try:
        raw = path.read_text(encoding="utf-8", errors="replace")
    except OSError as exc:
        return _error_result("html_parser", spec.type_name,
                             f"[Could not read {path.name}: {exc}]")

    # Prefer the already-battle-tested HTML extractor used by the browser
    # integration. Falls back to stripping tags via regex when unavailable.
    text: str
    try:
        from agntspace.integrations.browser import extract_readable_text
        text = extract_readable_text(raw) or ""
    except Exception:
        # Conservative stdlib fallback
        from html.parser import HTMLParser

        class _Extractor(HTMLParser):
            def __init__(self) -> None:
                super().__init__()
                self._skip_depth = 0
                self._buf: list[str] = []

            def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
                if tag in {"script", "style", "nav", "header", "footer", "aside"}:
                    self._skip_depth += 1

            def handle_endtag(self, tag: str) -> None:
                if tag in {"script", "style", "nav", "header", "footer", "aside"} and self._skip_depth > 0:
                    self._skip_depth -= 1

            def handle_data(self, data: str) -> None:
                if self._skip_depth == 0 and data.strip():
                    self._buf.append(data.strip())

            def result(self) -> str:
                return "\n".join(self._buf)

        p = _Extractor()
        p.feed(raw)
        text = p.result()

    # Peel frontmatter if present (some HTML files are Markdown-rendered
    # exports with frontmatter preamble).
    meta, body = _split_frontmatter(text or raw)

    return ExtractResult(
        elements=[Element(kind="paragraph", text=body.strip() or "[Empty HTML]")],
        doc_metadata=meta,
        media=[],
        provenance={},
    )


# ── Bundled specs ───────────────────────────────────────────────────────
#
# Phase 1 specs match `kb-ingest/config/source-types.yaml` exactly — same
# extensions, same prompt_focus, same `default_source_type` value. What
# changes is that each `(set_of_extensions, extractor)` combination gets
# its OWN FileTypeSpec, even when two specs share a prompt / source_type
# label. That's the cost of the cleaner registry shape — the YAML's
# "article" bucket spans md + html + txt + rst, but md and html need
# different extractors (text_utf8 vs html_parser), so the registry has
# two "article"-flavoured specs pointing at different extractors.
#
# When `_classify_source` returns `(spec.default_source_type, spec.prompt_focus)`,
# wiki frontmatter stays identical to the pre-registry world — a user's
# existing `.md` file and a freshly-ingested `.md` file both land with
# `source_type: article` + the same `prompt_focus`. Same for every other
# legacy type.
#
# PROMPT PARITY — these strings are copy-pasted from source-types.yaml.
# Do not edit here without updating both; YAML migration in Phase 2 will
# flip the dependency so YAML wins and this file goes away.

DEFAULT_SPECS: tuple[FileTypeSpec, ...] = (
    # ── article (md/txt/rst via text_utf8) ─────────────────────
    FileTypeSpec(
        type_name="article_text",
        display_name="Markdown / plain-text article",
        extensions=(".md", ".txt", ".rst"),
        mime_types=("text/markdown", "text/plain"),
        default_source_type="article",
        prompt_focus="Extract main arguments, key facts, quotes, author perspective.",
        extractor="text_utf8",
        canvas_content_type="markdown",
        ui_icon="FileText",
    ),
    # ── article (html/htm via html_parser) ─────────────────────
    FileTypeSpec(
        type_name="article_html",
        display_name="HTML article",
        extensions=(".html", ".htm"),
        mime_types=("text/html",),
        default_source_type="article",
        prompt_focus="Extract main arguments, key facts, quotes, author perspective.",
        extractor="html_parser",
        canvas_content_type="markdown",
        ui_icon="Globe",
    ),
    # ── paper (pdf) — Phase 4: multi-tier ladder ───────────────
    # Tier 1 (fast, default): pdfplumber — embedded text + font-cluster
    # heading detection + extract_tables(). Works for most "digital-native"
    # PDFs (papers, exports, docs with selectable text).
    # Tier 2 (capable): pdfplumber_ocr — same pipeline + page-level OCR
    # fallback via pytesseract for scanned pages that return empty text.
    # Costs extra per-page render + OCR but unlocks scanned-only content
    # that Tier 1 can only report as a sentinel.
    FileTypeSpec(
        type_name="paper_pdf",
        display_name="PDF document",
        extensions=(".pdf",),
        mime_types=("application/pdf",),
        magic_prefix=b"%PDF-",
        default_source_type="paper",
        prompt_focus="Extract thesis, methodology, key findings, citations. Academic rigor.",
        extractor="pdf_pdfplumber",   # Phase 1-3 compat — first-tier fallback
        extractors=(
            ExtractorBinding(
                extractor="pdf_pdfplumber",
                cost_tier="fast",
                fidelity="medium",
                description="pdfplumber — embedded text + layout + tables",
            ),
            ExtractorBinding(
                extractor="pdf_pdfplumber_ocr",
                cost_tier="capable",
                fidelity="high",
                description="pdfplumber with page-level OCR fallback for scanned pages",
            ),
        ),
        requires_binary=True,
        canvas_content_type="markdown",
        canvas_preview="thumbnail",
        ui_icon="FileText",
        max_bytes=10 * 1024 * 1024,
    ),
    # ── image (vision LLM) ─────────────────────────────────────
    FileTypeSpec(
        type_name="image_vision",
        display_name="Image (vision LLM + OCR)",
        extensions=(".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"),
        mime_types=("image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/webp"),
        default_source_type="image",
        prompt_focus="Describe what the image shows, any text/labels visible, context.",
        extractor="vision_llm",
        requires_binary=True,
        vision_capable=True,
        canvas_content_type="image",
        canvas_preview="inline",
        ui_icon="Image",
    ),
    # ── data (csv/tsv via csv_tabular) ─────────────────────────
    FileTypeSpec(
        type_name="data_csv",
        display_name="Tabular data (CSV/TSV)",
        extensions=(".csv", ".tsv"),
        mime_types=("text/csv", "text/tab-separated-values"),
        default_source_type="data",
        prompt_focus="Describe the data structure, key fields, notable patterns, size.",
        extractor="csv_tabular",
        canvas_content_type="table",
        ui_icon="Table",
    ),
    # ── data (yaml/yml via text_utf8) ──────────────────────────
    FileTypeSpec(
        type_name="data_yaml",
        display_name="YAML data",
        extensions=(".yaml", ".yml"),
        mime_types=("application/x-yaml", "text/x-yaml"),
        default_source_type="data",
        prompt_focus="Describe the data structure, key fields, notable patterns, size.",
        extractor="text_utf8",
        canvas_content_type="markdown",
        ui_icon="FileCode",
    ),
    # ── chat_history (jsonl via json_pretty) ───────────────────
    FileTypeSpec(
        type_name="chat_history_jsonl",
        display_name="Chat history (JSONL)",
        extensions=(".jsonl",),
        mime_types=("application/x-ndjson",),
        default_source_type="chat_history",
        prompt_focus="Extract key decisions, insights, topics discussed, action items from this conversation.",
        extractor="json_pretty",
        canvas_content_type="json",
        ui_icon="MessageSquare",
    ),
    # ── export (json via json_pretty) ──────────────────────────
    FileTypeSpec(
        type_name="export_json",
        display_name="JSON export",
        extensions=(".json",),
        mime_types=("application/json",),
        default_source_type="export",
        prompt_focus="Extract key topics, decisions, people mentioned, and actionable insights.",
        extractor="json_pretty",
        canvas_content_type="json",
        ui_icon="Braces",
    ),
    # ── transcript (vtt/srt) ───────────────────────────────────
    FileTypeSpec(
        type_name="transcript_subs",
        display_name="Subtitle / transcript",
        extensions=(".vtt", ".srt"),
        mime_types=("text/vtt", "application/x-subrip"),
        default_source_type="transcript",
        prompt_focus="Extract key discussion points, decisions made, action items, attendees mentioned.",
        extractor="vtt_srt",
        canvas_content_type="markdown",
        ui_icon="FileAudio",
    ),
    # ── code (py/js/ts via text_utf8) ──────────────────────────
    FileTypeSpec(
        type_name="code_text",
        display_name="Source code (Python/JS/TS)",
        extensions=(".py", ".js", ".ts"),
        mime_types=("text/x-python", "text/javascript", "application/typescript"),
        default_source_type="code",
        prompt_focus="Describe purpose, key functions, architecture, dependencies.",
        extractor="text_utf8",
        canvas_content_type="code",
        ui_icon="FileCode",
    ),
    # ── Phase 3: document formats ──────────────────────────────
    # ── docx (python-docx) ─────────────────────────────────────
    FileTypeSpec(
        type_name="document_docx",
        display_name="Microsoft Word document",
        extensions=(".docx",),
        mime_types=(
            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        ),
        magic_prefix=b"PK\x03\x04",  # DOCX is a ZIP archive
        default_source_type="article",
        prompt_focus="Extract main arguments, key facts, headings, tables, author perspective.",
        extractor="docx_python_docx",
        requires_binary=True,
        chunking_strategy="structural",
        canvas_content_type="markdown",
        canvas_preview="inline",
        ui_icon="FileText",
    ),
    # ── epub (ebooklib) ────────────────────────────────────────
    FileTypeSpec(
        type_name="book_epub",
        display_name="ePub e-book",
        extensions=(".epub",),
        mime_types=("application/epub+zip",),
        magic_prefix=b"PK\x03\x04",  # ePub is a ZIP archive
        default_source_type="article",
        prompt_focus="Extract main arguments, key chapters, author perspective, notable quotes.",
        extractor="epub_ebooklib",
        requires_binary=True,
        chunking_strategy="structural",
        canvas_content_type="markdown",
        canvas_preview="thumbnail",
        ui_icon="BookOpen",
        max_bytes=50 * 1024 * 1024,  # ePub can legitimately be larger than 10 MB
    ),
    # ── Phase 3: audio + video ─────────────────────────────────
    # ── audio (faster-whisper via llm.transcribe) ──────────────
    FileTypeSpec(
        type_name="audio_recording",
        display_name="Audio recording",
        extensions=(".mp3", ".wav", ".m4a", ".flac", ".ogg", ".opus"),
        mime_types=(
            "audio/mpeg", "audio/wav", "audio/x-wav",
            "audio/mp4", "audio/flac", "audio/ogg", "audio/opus",
        ),
        default_source_type="transcript",
        prompt_focus="Extract key discussion points, decisions made, action items, speakers mentioned.",
        extractor="audio_whisper",
        requires_binary=True,
        transcribe=True,
        canvas_content_type="markdown",
        canvas_preview="inline",
        ui_icon="FileAudio",
        max_bytes=50 * 1024 * 1024,
    ),
    # ── video (ffmpeg keyframe + vision + whisper) ─────────────
    FileTypeSpec(
        type_name="video_recording",
        display_name="Video recording",
        extensions=(".mp4", ".webm", ".mov", ".mkv"),
        mime_types=(
            "video/mp4", "video/webm", "video/quicktime", "video/x-matroska",
        ),
        default_source_type="transcript",
        prompt_focus="Describe visual content and extract spoken discussion, decisions, action items.",
        extractor="video_ffmpeg_vision",
        requires_binary=True,
        transcribe=True,
        vision_capable=True,
        canvas_content_type="markdown",
        canvas_preview="thumbnail",
        ui_icon="FileVideo",
        max_bytes=100 * 1024 * 1024,  # short clips; longer videos hit watcher cap naturally
    ),
)


def build_default_registry() -> FileTypeRegistry:
    """Produce a FileTypeRegistry with the 7 bundled specs + extractors.

    Called once at service init; Phase 2 augments this instance with
    per-type skill YAMLs via ``load_file_type_skills`` (see below).
    Plugins may register their own types after that.
    """
    reg = FileTypeRegistry()
    reg.register_extractor("text_utf8", extract_text_utf8)
    reg.register_extractor("pdf_pdfplumber", extract_pdf_pdfplumber)
    reg.register_extractor("vision_llm", extract_vision_llm)
    reg.register_extractor("csv_tabular", extract_csv_tabular)
    reg.register_extractor("json_pretty", extract_json_pretty)
    reg.register_extractor("vtt_srt", extract_vtt_srt)
    reg.register_extractor("html_parser", extract_html_parser)
    # Phase 3 — new formats
    reg.register_extractor("docx_python_docx", extract_docx_python_docx)
    reg.register_extractor("epub_ebooklib", extract_epub_ebooklib)
    reg.register_extractor("audio_whisper", extract_audio_whisper)
    reg.register_extractor("video_ffmpeg_vision", extract_video_ffmpeg_vision)
    # Phase 4 — second-tier extractors (registered before specs so the
    # ladder's extractor_id lookup succeeds at registration time).
    reg.register_extractor("pdf_pdfplumber_ocr", extract_pdf_pdfplumber_ocr)
    for spec in DEFAULT_SPECS:
        reg.register_type(spec)
    return reg


# ── Phase 2: YAML-driven skill loader ──────────────────────────────────


# Required fields in a `file-type.yaml`. Missing → skip with warning.
_REQUIRED_YAML_FIELDS: tuple[str, ...] = ("type_name", "extensions", "extractor")  # arch:deterministic — schema enforcement, not user-tunable policy


def _coerce_magic_prefix(value: Any) -> bytes | None:
    """YAML can't carry a Python bytes literal directly. The YAML file
    ships ``magic_prefix: "%PDF-"`` as a string; we latin-1 encode it
    (preserves all byte values 0-255) so the extractor path matches.
    ``None`` / missing / empty string → no magic-byte matching.
    """
    if value is None:
        return None
    if isinstance(value, bytes):
        return value
    if isinstance(value, str):
        return value.encode("latin-1") if value else None
    log.warning("file-types loader: unexpected magic_prefix type %s; ignoring",
                type(value).__name__)
    return None


def _spec_from_yaml(data: dict, *, source: str) -> FileTypeSpec | None:
    """Parse one `file-type.yaml` dict into a FileTypeSpec.

    Returns ``None`` when required fields are missing or malformed —
    the caller logs + continues with the next spec so ONE bad YAML
    doesn't break the whole registry bootstrap.
    """
    for field_name in _REQUIRED_YAML_FIELDS:
        if field_name not in data:
            log.warning("file-types loader: %s missing required field '%s'; skipping",
                        source, field_name)
            return None

    extensions = data.get("extensions") or []
    if not isinstance(extensions, list) or not extensions:
        log.warning("file-types loader: %s has empty/invalid extensions; skipping",
                    source)
        return None

    mime_types = data.get("mime_types") or []
    if not isinstance(mime_types, list):
        mime_types = []

    # Phase 4: optional extractor ladder. Entries are dicts with
    # {extractor, cost_tier, fidelity, description, options}. Malformed
    # entries inside an otherwise-valid ladder are skipped with a warning
    # rather than failing the whole spec — one bad ladder row shouldn't
    # kill a file-type's registration.
    raw_extractors = data.get("extractors") or []
    extractor_bindings: list[ExtractorBinding] = []
    if isinstance(raw_extractors, list):
        for idx, entry in enumerate(raw_extractors):
            if not isinstance(entry, dict):
                log.warning("file-types loader: %s extractors[%d] is not a dict; skipping", source, idx)
                continue
            eid = entry.get("extractor")
            if not eid or not isinstance(eid, str):
                log.warning("file-types loader: %s extractors[%d] missing 'extractor' id; skipping", source, idx)
                continue
            try:
                extractor_bindings.append(ExtractorBinding(
                    extractor=eid,
                    cost_tier=str(entry.get("cost_tier", "fast")),  # type: ignore[arg-type]
                    fidelity=str(entry.get("fidelity", "medium")),  # type: ignore[arg-type]
                    description=str(entry.get("description", "")),
                    options=dict(entry.get("options") or {}),
                ))
            except (TypeError, ValueError) as exc:
                log.warning("file-types loader: %s extractors[%d] coerce failed: %s; skipping",
                            source, idx, exc)
                continue

    try:
        spec = FileTypeSpec(
            type_name=str(data["type_name"]),
            display_name=str(data.get("display_name", data["type_name"])),
            extensions=tuple(str(e).lower() for e in extensions),
            mime_types=tuple(str(m).lower() for m in mime_types),
            magic_prefix=_coerce_magic_prefix(data.get("magic_prefix")),
            default_source_type=str(data.get("default_source_type", "article")),
            prompt_focus=str(data.get("prompt_focus", "Extract main points, key facts, and context.")),
            max_bytes=int(data.get("max_bytes", 10 * 1024 * 1024)),
            requires_binary=bool(data.get("requires_binary", False)),
            extractor=str(data["extractor"]),
            extractor_options=dict(data.get("extractor_options") or {}),
            extractors=tuple(extractor_bindings),
            chunking_strategy=str(data.get("chunking_strategy", "fixed")),  # type: ignore[arg-type]
            transcribe=bool(data.get("transcribe", False)),
            vision_capable=bool(data.get("vision_capable", False)),
            canvas_content_type=str(data.get("canvas_content_type", "markdown")),
            canvas_preview=str(data.get("canvas_preview", "inline")),  # type: ignore[arg-type]
            ui_icon=str(data.get("ui_icon", "FileText")),
            thumbnail_generator=data.get("thumbnail_generator"),
        )
    except (TypeError, ValueError) as exc:
        log.warning("file-types loader: %s failed to coerce fields: %s; skipping",
                    source, exc)
        return None

    return spec


def load_file_type_skills(
    registry: FileTypeRegistry,
    skill_loader: object,
) -> int:
    """Read every ``file-type.yaml`` under the skill_loader's tree and
    register the resulting specs with ``overwrite=True`` so user vault
    edits win over bundled Python defaults.

    Returns the number of specs loaded + registered successfully. Never
    raises — malformed YAMLs are skipped with a warning so the registry
    is never left in a broken state. The skill_loader is typed as
    ``object`` to keep this module decoupled from ``skills.loader`` —
    the only attributes we touch are ``.skills`` (dict name→Skill) and
    ``Skill.category`` + ``Skill.path``.
    """
    loaded = 0
    try:
        skills = getattr(skill_loader, "skills", None) or {}
    except Exception:
        log.debug("file-types loader: skill_loader.skills access failed", exc_info=True)
        return 0

    # Use SkillLoader's own `load_skill_config_file` to resolve paths —
    # it handles the SKILL.md-is-relative quirk (stored path is a relative
    # string vs the skills_dir root) and applies whatever vault-override
    # semantics the loader enforces in future. Falls back gracefully when
    # the method isn't available (some test stubs pass a minimal object).
    load_cfg = getattr(skill_loader, "load_skill_config_file", None)

    for skill_name, skill in skills.items():
        category = getattr(skill, "category", "")
        if category != "file-types":
            continue

        data: dict | None = None
        if callable(load_cfg):
            try:
                loaded_obj = load_cfg(skill_name, "file-type.yaml")
                if isinstance(loaded_obj, dict):
                    data = loaded_obj
            except Exception as exc:
                log.warning("file-types loader: %s load_skill_config_file raised: %s",
                            skill_name, exc)

        # Fallback: when the skill_loader doesn't expose load_skill_config_file
        # (e.g. a minimal stub in tests), resolve the path manually under the
        # loader's own skills_dir.
        if data is None:
            skill_path = getattr(skill, "path", None)
            skills_dir = getattr(skill_loader, "_skills_dir", None)
            if not (skill_path and skills_dir):
                continue
            skill_rel = Path(skill_path)
            if skill_rel.name == "SKILL.md":
                skill_rel = skill_rel.parent
            config_path = Path(skills_dir) / skill_rel / "config" / "file-type.yaml"
            if not config_path.is_file():
                log.debug("file-types loader: %s missing config/file-type.yaml", skill_name)
                continue
            try:
                import yaml as _yaml
                data = _yaml.safe_load(config_path.read_text(encoding="utf-8"))
            except Exception as exc:
                log.warning("file-types loader: %s YAML parse failed: %s", config_path, exc)
                continue

        if not isinstance(data, dict):
            log.warning("file-types loader: %s returned non-dict YAML; skipping", skill_name)
            continue

        spec = _spec_from_yaml(data, source=skill_name)
        if spec is None:
            continue

        try:
            registry.register_type(spec, overwrite=True)
            loaded += 1
            log.debug("file-types loader: registered %s from skill %s",
                      spec.type_name, skill_name)
        except Exception as exc:
            log.warning("file-types loader: failed to register %s: %s",
                        spec.type_name, exc)

    if loaded:
        log.info("FileTypeRegistry: loaded %d spec(s) from file-type skills", loaded)
    return loaded
