"""Inbox folder watcher — auto-ingests files dropped into inbox/.

Watches both the global inbox/ (vault root) and per-KB inbox/ directories.
When new files appear, triggers ingest automatically.
Checks every 30 seconds.
"""

from __future__ import annotations

import asyncio
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from agntspace.agent.orchestrator import Orchestrator
    from agntspace.automation.heartbeat import Heartbeat
    from agntspace.kb.service import KnowledgeBaseService

log = logging.getLogger(__name__)


class RawWatcher:
    """Watches inbox/ and root folder for new files to ingest."""

    _MAX_RETRIES = 5

    def __init__(
        self,
        vault_dir: Path,
        kb_service: KnowledgeBaseService,
        root_dir: Path | None = None,
        scan_root: bool = False,
        heartbeat: Heartbeat | None = None,
        orchestrator: Orchestrator | None = None,
        check_interval: int = 60,
        work_queue: object | None = None,
        settings_service: object | None = None,
    ) -> None:
        self._vault_dir = vault_dir
        self._paths = None  # Set via watcher._paths = vault_paths in main.py
        self._root_dir = root_dir if scan_root else None
        self._kb = kb_service
        self._heartbeat = heartbeat
        self._orchestrator = orchestrator
        self._interval = check_interval
        self._work_queue = work_queue
        # User-controlled pause flag (config.toml [watcher].paused). Read on every
        # loop tick + check_and_ingest entry so toggling via Settings takes effect
        # immediately without a restart. Default False — watcher runs on fresh
        # installs, matching pre-pause behavior.
        self._settings_service = settings_service
        # Agent-mediated flows (Rule #17): when wired, every ingest goes
        # through ``agent.invoke_skill`` instead of ``kb.ingest()`` direct.
        # main.py sets both ``_agent`` and ``_agent_mediated`` after start.
        # When either is unset, the watcher falls back to direct calls —
        # that's the safety valve for minimal installs and tests.
        self._agent: object | None = None
        self._agent_mediated: bool = False
        self._task: asyncio.Task | None = None  # type: ignore[type-arg]
        self._running = False
        self._known_files: set[str] = set()
        # Track root folder files: path → content hash (for rename/move/delete detection)
        self._root_file_hashes: dict[str, str] = {}
        # Track file mtime at time of ingestion — detect content modifications
        self._known_mtimes: dict[str, float] = {}
        # Track file sizes at time of ingestion — detect content added after creation
        self._known_sizes: dict[str, int] = {}
        # Track failed ingest attempts per file — stop retrying after _MAX_RETRIES
        self._fail_counts: dict[str, int] = {}
        # Prevent concurrent ingest runs (watcher cycle vs manual API call)
        self._ingesting = False
        # Observability: track last cycle for /api/watcher/status
        self._last_cycle_at: float = 0.0
        self._last_cycle_ingested: int = 0
        self._last_cycle_deferred: int = 0
        self._last_error: str = ""
        self._cycle_count: int = 0
        # Event-driven wake-up: watchdog sets this to wake the poll loop
        # immediately on a filesystem event. Poll loop also wakes on its
        # own every `_interval` seconds as a safety net.
        self._event_drain: asyncio.Event = asyncio.Event()
        # Last time a watchdog event woke us up (for observability).
        self._last_wake_at: float = 0.0
        self._wake_count: int = 0

    def _resolve(self, relative_path: str) -> Path:
        """Resolve a logical vault path to a physical path."""
        if self._paths:
            return self._paths.resolve(relative_path)
        return self._vault_dir / relative_path

    @property
    def _state_path(self) -> Path:
        return self._vault_dir / ".inbox-watcher-state.json"

    def _load_state(self) -> set[str]:
        """Load known files from disk so we don't re-process across restarts.

        Catch-up: inbox files that were marked known but never actually
        produced a library copy are dropped from the known set so the
        watcher retries them.  This fixes the case where a previous
        ingest failed (e.g. LLM was down) and the file got stuck.
        """
        try:
            if self._state_path.exists():
                data = json.loads(self._state_path.read_text(encoding="utf-8"))
                # Only keep entries for files that still exist
                known = {f for f in data.get("known_files", []) if Path(f).exists()}

                # Catch-up: inbox files without library evidence are orphans
                orphans: set[str] = set()
                kb_prefix = str(self._resolve("knowledge"))
                for fpath in known:
                    fpath_posix = fpath.replace("\\", "/")
                    if "/inbox/" not in fpath_posix:
                        continue
                    p = Path(fpath)
                    if not p.exists():
                        continue
                    # Resolve KB slug from path
                    if fpath.startswith(kb_prefix) or fpath.replace("\\", "/").startswith(kb_prefix.replace("\\", "/")):
                        try:
                            rel = p.relative_to(self._resolve("knowledge"))
                            slug = str(rel.parts[0])
                            lib_dir = self._resolve(f"knowledge/{slug}/library")
                            if lib_dir.is_dir() and any(
                                lf.stem == p.stem or lf.name == p.name
                                for lf in lib_dir.rglob("*") if lf.is_file()
                            ):
                                continue  # has library evidence — genuinely processed
                            orphans.add(fpath)
                        except (ValueError, IndexError):
                            pass

                if orphans:
                    log.info(
                        "Inbox watcher: %d orphaned inbox file(s) detected on startup — will retry: %s",
                        len(orphans),
                        ", ".join(Path(f).name for f in orphans),
                    )
                    known -= orphans

                # Restore persisted sizes and mtimes
                self._known_sizes = {
                    k: v for k, v in data.get("known_sizes", {}).items()
                    if k in known
                }
                self._known_mtimes = {
                    k: v for k, v in data.get("known_mtimes", {}).items()
                    if k in known
                }
                return known
        except Exception:
            log.debug("Could not load watcher state, starting fresh")
        return set()

    def _save_state(self) -> None:
        """Persist known files and their sizes to disk."""
        try:
            self._state_path.write_text(
                json.dumps({
                    "known_files": list(self._known_files),
                    "known_sizes": self._known_sizes,
                    "known_mtimes": self._known_mtimes,
                }),
                encoding="utf-8",
            )
        except Exception:
            log.debug("Could not save watcher state", exc_info=True)

    def start(self) -> None:
        if self._running:
            return
        self._running = True
        self._known_files = self._load_state()
        self._task = asyncio.create_task(self._run())
        log.debug("Inbox watcher started (interval: %ds, %d known files)", self._interval, len(self._known_files))

    async def _run(self) -> None:
        """Initial check + regular loop."""
        # Small delay to let other services initialize
        await asyncio.sleep(5)
        # Process any files already in inbox/ (dropped while server was down)
        try:
            result = await self.check_and_ingest()
            if result:
                log.info("Inbox watcher: startup ingest processed %d files", len(result))
        except Exception:
            log.exception("Inbox watcher: startup ingest failed")
        await self._loop()

    def stop(self) -> None:
        self._running = False
        if self._task:
            self._task.cancel()
            self._task = None

    def trigger(self) -> None:
        """Wake the poll loop immediately. Safe to call from any thread.

        Used by the watchdog event handler to translate filesystem events
        into watcher cycles without blocking.
        """
        try:
            self._event_drain.set()
        except Exception:
            log.debug("trigger: could not set drain event", exc_info=True)

    def is_paused(self) -> bool:
        """True when the user has paused automatic ingestion via Settings.

        Reads ``[watcher].paused`` from config.toml on every call so toggling
        in the UI takes effect on the next loop tick without a restart. Defaults
        to False (watcher runs) when the settings service is missing or the
        key is absent — matching pre-pause behavior on every existing vault.
        """
        svc = self._settings_service
        if svc is None:
            return False
        try:
            return bool(svc.get_value("watcher", "paused", False))
        except Exception:
            return False

    def get_status(self) -> dict:
        """Return current watcher state for observability (/api/watcher/status)."""
        import time
        now = time.time()
        state_path = self._state_path
        state_exists = state_path.is_file()
        state_mtime = state_path.stat().st_mtime if state_exists else 0

        # Count quarantined files across all KBs
        quarantine_count = 0
        try:
            kb_dir = self._resolve("knowledge")
            if kb_dir.is_dir():
                for kb in kb_dir.iterdir():
                    q = kb / "inbox" / ".quarantine"
                    if q.is_dir():
                        quarantine_count += sum(1 for _ in q.iterdir() if _.is_file())
        except Exception:
            pass

        # Optional fs_events bridge (watchdog). None if not wired up.
        fs_events_status: dict | None = None
        bridge = getattr(self, "_fs_bridge", None)
        if bridge is not None:
            try:
                fs_events_status = bridge.status()
            except Exception:
                fs_events_status = {"available": False, "running": False}

        return {
            "running": self._running,
            "paused": self.is_paused(),
            "interval_seconds": self._interval,
            "known_files": len(self._known_files),
            "fail_counts": sum(self._fail_counts.values()),
            "fail_tracked": len(self._fail_counts),
            "quarantine_count": quarantine_count,
            "cycle_count": self._cycle_count,
            "last_cycle_at": self._last_cycle_at,
            "last_cycle_seconds_ago": (now - self._last_cycle_at) if self._last_cycle_at else None,
            "last_cycle_ingested": self._last_cycle_ingested,
            "last_cycle_deferred": self._last_cycle_deferred,
            "last_error": self._last_error,
            "state_file": str(state_path),
            "state_file_exists": state_exists,
            "state_file_mtime": state_mtime,
            # Real-time event bridge
            "wake_count": self._wake_count,
            "last_wake_at": self._last_wake_at,
            "last_wake_seconds_ago": (now - self._last_wake_at) if self._last_wake_at else None,
            "fs_events": fs_events_status,
        }

    # Directories to skip when scanning root folder
    _SKIP_DIRS = {  # arch:deterministic — mirrors fs_events._SKIP_DIR_NAMES; structural vault internals + universal OS noise, not user-tunable
        # AgntSpace / vault internals (old + new layout)
        "agntspace", "agntspace-inbox", "agntspace-journal",
        "agntspace-knowledge", "agntspace-projects",
        "agntspace-goals",
        ".obsidian", ".git", ".trash",
        # Dev / system
        "node_modules", "__pycache__", ".venv", "venv", ".env",
        # OS system dirs (when root is home dir)
        "AppData", "Application Data", ".cache", ".local", ".config", ".vscode",
        "Library", ".Trash",
        # Cloud sync (managed separately if user connects them)
        "OneDrive", "Dropbox", "Google Drive", "iCloud Drive",
        # Common large dirs
        "Downloads", ".npm", ".nuget", ".cargo", ".rustup",
    }

    _SUPPORTED_EXTS = {  # arch:deterministic — capability surface of the ingest pipeline, mirrors fs_events._SUPPORTED_EXTS
        # Text + web (text_utf8 / html_parser)
        ".md", ".txt", ".html", ".htm", ".rst",
        # Documents (pdf / docx / epub)
        ".pdf", ".docx", ".epub",
        # Images (vision_llm)
        ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp",
        # Tabular / structured (csv_tabular / json_pretty / text_utf8 for yaml)
        ".csv", ".tsv", ".yaml", ".yml", ".json", ".jsonl",
        # Transcripts (vtt_srt)
        ".vtt", ".srt",
        # Code (text_utf8)
        ".py", ".js", ".ts",
        # Audio (audio_whisper)
        ".mp3", ".wav", ".m4a", ".flac", ".ogg", ".opus",
        # Video (video_ffmpeg_vision)
        ".mp4", ".webm", ".mov", ".mkv",
    }

    def _scan_all_raw(self) -> set[str]:
        """Scan inbox/ directories and root folder for files."""
        files: set[str] = set()

        # 1. agntspace/inbox/ (explicit drop zone)
        inbox = self._resolve("inbox")
        if inbox.is_dir():
            for f in inbox.iterdir():
                if f.is_file() and f.name != ".gitkeep":
                    files.add(str(f))

        # 2. Per-KB inbox/
        kb_dir = self._resolve("knowledge")
        if kb_dir.is_dir():
            for kb in kb_dir.iterdir():
                if kb.is_dir():
                    kb_inbox = kb / "inbox"
                    if kb_inbox.is_dir():
                        for f in kb_inbox.iterdir():
                            if f.is_file() and f.name != ".gitkeep":
                                files.add(str(f))

        # 3. Root folder — scan for user content to ingest as knowledge
        #    agntspace/ is inside root, so root IS the vault
        if self._root_dir and self._root_dir != self._vault_dir:
            self._scan_dir_recursive(self._root_dir, files)

        return files

    def _scan_dir_flat(self, directory: Path, files: set[str]) -> None:
        """Scan only top-level files in a directory (non-recursive)."""
        try:
            for f in directory.iterdir():
                if f.is_file() and not f.name.startswith(".") and f.suffix.lower() in self._SUPPORTED_EXTS:
                    files.add(str(f))
        except OSError:
            pass

    # How many seconds a file must exist before we process it.
    # Editors like Obsidian create an empty file first, then write content
    # moments later.  Deferring very young files avoids ingesting the blank
    # shell and marking the file as "known" before the real content arrives.
    # At 30-second scan intervals, a 5-second grace period means the file
    # is picked up on the same cycle or the next one — no perceptible delay.
    _SETTLE_SECONDS = 5
    # Skip files larger than 10MB — likely binaries or exports that would
    # blow up LLM token budgets and chunked extraction runtime.
    _MAX_FILE_BYTES = 10 * 1024 * 1024

    def _scan_dir_recursive(self, directory: Path, files: set[str]) -> None:
        """Recursively scan a directory, skipping system/hidden dirs."""
        import time

        now = time.time()
        try:
            for f in directory.iterdir():
                if f.name.startswith(".") or f.name in self._SKIP_DIRS:
                    continue
                if f.is_dir():
                    self._scan_dir_recursive(f, files)
                elif f.is_file() and f.suffix.lower() in self._SUPPORTED_EXTS:
                    try:
                        st = f.stat()
                    except OSError:
                        continue
                    # Skip empty files (0 bytes is never useful)
                    if st.st_size == 0:
                        continue
                    # Skip huge files (binaries, data dumps) — would blow up LLM budget
                    if st.st_size > self._MAX_FILE_BYTES:
                        log.debug("Skipping oversized file (%d bytes): %s",
                                  st.st_size, f.name)
                        continue
                    # Defer very young files — let the editor finish writing
                    if now - st.st_mtime < self._SETTLE_SECONDS:
                        continue
                    files.add(str(f))
        except OSError:
            pass  # Skip inaccessible dirs (OneDrive placeholders, permissions)

    # Image extensions that can be referenced by markdown files.
    _IMAGE_EXTS = frozenset({  # arch:deterministic — image format capabilities of the vision pipeline
        ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".tiff", ".tif",
    })

    def _copy_sibling_images(self, md_file: Path, dest_dir: Path) -> int:
        """Copy images referenced by *md_file* into *dest_dir*.

        Parses ``![alt](path)`` and ``![[path]]`` references, resolves
        them relative to the markdown file's parent directory (and common
        attachment subdirs like ``assets/``, ``attachments/``, ``images/``),
        and copies any that exist.  Returns the number of images copied.

        This is the companion to ``kb.image_resolver.resolve_local_images``
        but runs at the watcher layer — before the file reaches the inbox —
        so images travel alongside their markdown.
        """
        try:
            from agntspace.kb.image_resolver import find_markdown_image_refs
        except ImportError:
            return 0

        try:
            text = md_file.read_text(encoding="utf-8", errors="replace")
        except OSError:
            return 0

        refs = find_markdown_image_refs(text)
        if not refs:
            return 0

        source_dir = md_file.parent
        attachment_subdirs = ["", "assets", "attachments", "images", "files", "media"]
        copied = 0

        for ref in refs:
            path_str = ref["path"]

            # Skip remote URLs — the ingest pipeline downloads those itself.
            if path_str.startswith(("http://", "https://", "data:")):
                continue

            # Skip non-image extensions
            ext = Path(path_str).suffix.lower()
            if ext not in self._IMAGE_EXTS:
                continue

            # Search source dir + common attachment subdirs
            found: Path | None = None
            for subdir in attachment_subdirs:
                candidate = source_dir / subdir / path_str if subdir else source_dir / path_str
                try:
                    if candidate.exists() and candidate.is_file():
                        found = candidate
                        break
                except OSError:
                    continue

            if not found:
                continue

            try:
                st = found.stat()
                if st.st_size == 0 or st.st_size > self._MAX_FILE_BYTES:
                    continue
            except OSError:
                continue

            dest = dest_dir / found.name
            try:
                if not dest.exists():
                    import shutil as _shutil
                    _shutil.copy2(str(found), str(dest))
                    copied += 1
                    log.debug("Copied sibling image %s alongside %s", found.name, md_file.name)
            except OSError:
                log.debug("Could not copy sibling image %s", found.name, exc_info=True)

        if copied:
            log.info("Watcher: copied %d sibling image(s) for %s", copied, md_file.name)
        return copied

    def _snapshot_file_stats(self, files: set[str]) -> None:
        """Record size and mtime for all files so we detect future changes."""
        for fpath in files:
            try:
                st = Path(fpath).stat()
                self._known_sizes[fpath] = st.st_size
                self._known_mtimes[fpath] = st.st_mtime
            except OSError:
                pass

    async def _ingest_via_agent(self, slug: str) -> dict:
        """Run KB ingest for ``slug`` through the full agent pipeline.

        When ``_agent_mediated`` is enabled AND an Agent is wired, calls
        ``agent.invoke_skill("kb-ingest", {"slug": slug})`` so the work
        flows through the librarian subagent, the skill tool grants, and
        the mutation counter. The counter catches hallucinated success
        (small-model LLMs narrating without calling real tools).

        On ``SkillZeroEffect``, enqueues to the persistent work queue
        instead of falling back to a direct service call — the whole
        point is to surface failure, not route around it. The method
        still returns a best-effort ``{"ingested": [], "skipped": []}``
        dict so the caller doesn't see an exception.

        When agent-mediated flows are off (legacy/minimal install), this
        falls straight through to ``kb.ingest()`` — that's the safety
        valve for migrations.
        """
        if self._agent_mediated and self._agent is not None:
            try:
                from agntspace.agent.agent import SkillZeroEffect  # lazy to avoid cycle
                # Timeout: local models (27B+) doing multi-step tool calls
                # can hang for 10+ minutes. Fall back to direct ingest.
                _timeout = 180  # 3 minutes — generous for cloud, tight for local
                outcome = await asyncio.wait_for(
                    self._agent.invoke_skill(  # type: ignore[attr-defined]
                        "kb-ingest",
                        {"slug": slug},
                        agent_name="librarian",
                        caller="watcher",
                    ),
                    timeout=_timeout,
                )
                # Successful agent-mediated run. The real mutations
                # landed via the librarian's tool calls — we don't
                # replay them. Return the counter summary so caller
                # logs still get meaningful numbers.
                return {
                    "slug": slug,
                    "ingested": [],  # populated via activity events, not here
                    "skipped": [],
                    "count": outcome.mutations.get("tool_calls", 0),
                    "via": "agent",
                    "mutations": outcome.mutations,
                }
            except SkillZeroEffect as exc:
                # The LLM said "done!" with no verifiable effect.
                # Enqueue for retry and let the user see the failure
                # in /decisions and the action plan.
                log.warning(
                    "kb-ingest via agent produced zero effect for %s — enqueuing retry (%s)",
                    slug, exc.mutations,
                )
                if self._work_queue is not None:
                    try:
                        await self._work_queue.enqueue(  # type: ignore[attr-defined]
                            "ingest", {"slug": slug}, deduplicate=True,
                        )
                    except Exception:
                        log.debug("work queue enqueue failed", exc_info=True)
                return {
                    "slug": slug,
                    "ingested": [],
                    "skipped": [],
                    "count": 0,
                    "via": "agent",
                    "error": "zero_effect",
                    "mutations": exc.mutations,
                }
            except TimeoutError:
                log.warning(
                    "kb-ingest via agent timed out for %s after %ds — falling back to direct ingest",
                    slug, _timeout,
                )
                # Fall through to direct path below instead of enqueuing
                pass
            except Exception:
                log.exception("kb-ingest via agent raised — enqueuing retry")
                if self._work_queue is not None:
                    try:
                        await self._work_queue.enqueue(  # type: ignore[attr-defined]
                            "ingest", {"slug": slug}, deduplicate=True,
                        )
                    except Exception:
                        log.debug("work queue enqueue failed", exc_info=True)
                return {
                    "slug": slug,
                    "ingested": [],
                    "skipped": [],
                    "count": 0,
                    "via": "agent",
                    "error": "dispatch_failed",
                }

        # Legacy path — direct service call. Used when agent-mediated
        # flows are off, when the agent isn't wired (tests, minimal
        # installs), or during staged rollouts. Callers see the same
        # dict shape either way.
        result = await self._kb.ingest(slug)
        result.setdefault("via", "direct")
        return result

    async def check_and_ingest(self) -> list[str]:
        """Check for new files and auto-ingest them.

        Wraps the cycle in a correlation scope so every activity event
        and decision produced by the ingest pipeline shares the same
        causal ID (Rule #13). The scope prefix is ``watcher`` so the
        origin is clear in the /decisions page.
        """
        if self.is_paused():
            log.debug("Inbox watcher: paused via Settings, skipping cycle")
            return []
        if self._ingesting:
            log.debug("Inbox watcher: ingest already in progress, skipping cycle")
            return []
        self._ingesting = True
        try:
            from agntspace.activity.decisions import (
                correlation_scope,
                new_correlation_id,
            )
            with correlation_scope(new_correlation_id("watcher")):
                return await self._do_check_and_ingest()
        finally:
            self._ingesting = False

    # Max files to process per watcher cycle. Prevents blocking for minutes
    # when a large vault is first connected. Remaining files are picked up
    # on subsequent 30-second cycles.
    #
    # Phase 2 (Rule 12): the cap is loaded from
    # ``defaults/skills/core/kb-ingest/config/throughput.yaml`` via
    # ``self._kb._load_throughput_cap("max_per_cycle", ...)`` so the
    # watcher and the kb-service share one source of truth and the
    # cap is user-tunable without code changes. The fallback below is
    # used only when the kb service / skill loader / config is missing.
    _FALLBACK_MAX_PER_CYCLE = 10  # arch:deterministic -- degraded-mode mirror of throughput.yaml; only used when the skill loader is unavailable, see Phase 2 docstring

    # Per-cycle file cap applied while the server is still booting (graph build
    # in progress, app.state.ready == False). The default ingest cap (10) plus
    # the backlog of 1000+ files on a full vault causes the ingest loop to
    # monopolize the event loop for the entire startup, keeping the UI
    # unresponsive for ~85 minutes. Throttling to 2/cycle during boot leaves
    # CPU + LLM bandwidth available for the background graph build to finish
    # (typically in 2-3 minutes), after which the cap reverts to the full
    # steady-state value from the skill config. Zero impact on steady-state
    # throughput — this only bites during the first few minutes post-restart.
    _BOOTING_MAX_PER_CYCLE = 2  # arch:deterministic — boot throttle (event-loop yielding), pure infrastructure

    @property
    def _max_per_cycle(self) -> int:
        # Boot-aware throttle: while the server is still warming up the
        # knowledge graph, yield CPU + LLM bandwidth by capping ingest at a
        # tiny batch. Falls back silently to the normal cap when no app
        # handle is wired (tests, minimal installs, or post-ready).
        app = getattr(self, "_app", None)
        if app is not None:
            try:
                if not bool(getattr(app.state, "ready", True)):
                    return self._BOOTING_MAX_PER_CYCLE
            except Exception:
                pass

        kb = getattr(self, "_kb", None)
        if kb is not None and hasattr(kb, "_load_throughput_cap"):
            try:
                return kb._load_throughput_cap("max_per_cycle", default=self._FALLBACK_MAX_PER_CYCLE)
            except Exception:
                pass
        return self._FALLBACK_MAX_PER_CYCLE

    def _unsupported_reason(self, f: Path) -> str | None:
        """Return a reason string if ``f`` is unsupported, else None.

        Called for files sitting in explicit drop-zones (global inbox or
        per-KB inbox). Root-folder files are already filtered by extension
        in ``_scan_dir_recursive`` — this method only fires for files the
        user actively placed in a drop-zone that the pipeline can't accept.
        """
        try:
            st = f.stat()
        except OSError:
            return None
        if st.st_size == 0:
            return None  # 0-byte = "not ready", not "unsupported"
        import time
        if time.time() - st.st_mtime < self._SETTLE_SECONDS:
            return None  # too young — let it settle
        if st.st_size > self._MAX_FILE_BYTES:
            return f"oversized:{st.st_size}"
        if f.suffix.lower() not in self._SUPPORTED_EXTS:
            return f"unsupported_type:{f.suffix.lower() or 'no_extension'}"
        return None

    async def _quarantine_unsupported_files(self) -> list[str]:
        """Park unsupported / oversized files in ``library/unsupported/``.

        Runs at the start of every cycle. Drops a sidecar ``.reason.txt``
        next to each parked file so the user knows WHY it was skipped,
        and emits a medium-importance activity event through the KB
        service so the fact is visible in /decisions and the activity
        feed (Rule #13 — no silent user actions).
        """
        parked: list[str] = []

        # 1. Global inbox → infer target KB (single-KB → that one, else general)
        global_inbox = self._resolve("inbox")
        if global_inbox.is_dir():
            global_candidates: list[tuple[Path, str]] = []
            for f in global_inbox.iterdir():
                if not f.is_file() or f.name == ".gitkeep":
                    continue
                reason = self._unsupported_reason(f)
                if reason:
                    global_candidates.append((f, reason))
            if global_candidates:
                kbs = self._kb.list_kbs()
                if len(kbs) == 1:
                    slug = kbs[0]["slug"]
                else:
                    slug = "general"
                    if not any(k["slug"] == slug for k in kbs):
                        try:
                            self._kb.create_kb("General")
                        except Exception:
                            log.debug("Could not auto-create 'general' KB", exc_info=True)
                for f, reason in global_candidates:
                    try:
                        dest = await self._kb.park_unsupported_file(slug, f, reason)
                        if dest:
                            parked.append(str(dest))
                    except Exception:
                        log.debug("Failed to park global inbox file %s", f.name, exc_info=True)

        # 2. Per-KB inboxes
        kb_dir = self._resolve("knowledge")
        if kb_dir.is_dir():
            for kb in kb_dir.iterdir():
                if not kb.is_dir():
                    continue
                kb_inbox = kb / "inbox"
                if not kb_inbox.is_dir():
                    continue
                for f in kb_inbox.iterdir():
                    if not f.is_file() or f.name == ".gitkeep":
                        continue
                    # Skip the .quarantine subdir contents already handled elsewhere
                    if f.parent.name == ".quarantine":
                        continue
                    reason = self._unsupported_reason(f)
                    if reason:
                        try:
                            dest = await self._kb.park_unsupported_file(kb.name, f, reason)
                            if dest:
                                parked.append(str(dest))
                        except Exception:
                            log.debug("Failed to park KB inbox file %s/%s", kb.name, f.name, exc_info=True)

        return parked

    async def _do_check_and_ingest(self) -> list[str]:
        """Internal implementation of check_and_ingest (holds the lock)."""
        # Park unsupported / oversized inbox files BEFORE the scan so the
        # pipeline never sees them. Keeps the inbox clean and gives the
        # user a visible event instead of silent skip.
        try:
            await self._quarantine_unsupported_files()
        except Exception:
            log.exception("Inbox watcher: unsupported-file quarantine failed")

        current_files = self._scan_all_raw()
        new_files = current_files - self._known_files

        # Detect known files whose content has changed (mtime differs).
        # This covers: user edits a file in Obsidian, or a file was ingested
        # while empty and now has content.  The KB ingest pipeline handles
        # updates (replaces old source summary + produced pages).
        for fpath in current_files & self._known_files:
            if fpath in self._fail_counts and self._fail_counts[fpath] >= self._MAX_RETRIES:
                continue
            try:
                st = Path(fpath).stat()
            except OSError:
                continue
            old_mtime = self._known_mtimes.get(fpath)
            old_size = self._known_sizes.get(fpath, -1)
            if old_mtime is not None and st.st_mtime > old_mtime:
                # File was modified since last ingest
                log.info("Inbox watcher: file modified (mtime %s → %s), re-processing: %s",
                         old_mtime, st.st_mtime, Path(fpath).name)
                new_files.add(fpath)
                self._known_files.discard(fpath)
            elif old_size == 0 and st.st_size > 0:
                # Legacy: was recorded at 0 bytes, now has content
                log.info("Inbox watcher: previously-empty file now has content (%d bytes), re-processing: %s",
                         st.st_size, Path(fpath).name)
                new_files.add(fpath)
                self._known_files.discard(fpath)

        # Filter out files that have exceeded retry limit — queue to persistent work queue
        exhausted = {f for f in new_files if self._fail_counts.get(f, 0) >= self._MAX_RETRIES}
        if exhausted:
            log.warning("Inbox watcher: %d files exceeded %d retries — queuing for persistent retry: %s",
                        len(exhausted), self._MAX_RETRIES,
                        ", ".join(Path(f).name for f in list(exhausted)[:5]))
            # Queue exhausted files to the persistent work queue (retries forever)
            if hasattr(self, '_work_queue') and self._work_queue:
                for fpath in exhausted:
                    try:
                        # Determine which KB this file belongs to
                        p = Path(fpath)
                        kb_prefix = str(self._resolve("knowledge"))
                        if fpath.startswith(kb_prefix):
                            rel = p.relative_to(self._resolve("knowledge"))
                            slug = rel.parts[0]
                        else:
                            slug = "general"
                        asyncio.create_task(self._work_queue.enqueue(
                            "ingest", {"slug": slug}, deduplicate=True,
                        ))
                    except Exception:
                        log.debug("Inbox watcher: failed to queue %s", fpath, exc_info=True)
            self._known_files |= exhausted  # treat as known so they stop appearing
            new_files -= exhausted

        if not new_files:
            # Nothing new — update full state
            self._known_files = current_files
            self._snapshot_file_stats(current_files)
            self._save_state()
            return []

        # Limit per cycle to avoid blocking for minutes on large vaults.
        # Unprocessed files stay "new" and are picked up next cycle.
        max_per_cycle = self._max_per_cycle
        if len(new_files) > max_per_cycle:
            batch = set(sorted(new_files)[:max_per_cycle])
            deferred = new_files - batch
            log.info("Inbox watcher: %d new files, processing %d this cycle (%d deferred)",
                     len(new_files), len(batch), len(deferred))
            new_files = batch
        else:
            deferred = set()

        ingested = []

        # Group by location: inbox files, KB inbox files, root folder files
        inbox_new = []
        kb_new: dict[str, list[str]] = {}
        root_new: list[Path] = []

        inbox_prefix = str(self._resolve("inbox"))
        kb_prefix = str(self._resolve("knowledge"))

        for fpath in new_files:
            p = Path(fpath)
            fpath_str = str(p)
            if fpath_str.startswith(inbox_prefix):
                inbox_new.append(p.name)
            elif fpath_str.startswith(kb_prefix):
                # Extract KB slug
                try:
                    rel = p.relative_to(self._resolve("knowledge"))
                    slug = rel.parts[0]
                    kb_new.setdefault(slug, []).append(p.name)
                except (ValueError, IndexError):
                    pass
            else:
                # Root folder file — user's own content
                root_new.append(p)

        import shutil

        slugs_to_compile: set[str] = set()

        # Process inbox/ files (explicit drop zone — files get MOVED)
        if inbox_new:
            log.info("Inbox watcher: %d new files in inbox/: %s", len(inbox_new), ", ".join(inbox_new[:5]))
            try:
                kbs = self._kb.list_kbs()
                slug = "general"
                if not any(k["slug"] == slug for k in kbs):
                    self._kb.create_kb("General")

                kb_inbox = self._resolve(f"knowledge/{slug}/inbox")
                kb_inbox.mkdir(parents=True, exist_ok=True)
                for fname in inbox_new:
                    src = self._resolve("inbox") / fname
                    if not src.exists():
                        continue
                    dest = kb_inbox / fname
                    if dest.exists():
                        src.unlink()  # already in KB inbox
                    else:
                        shutil.move(str(src), str(dest))

                # Route through Agent.invoke_skill when available
                # (Rule #17). Direct fallback on legacy / minimal installs.
                result = await self._ingest_via_agent(slug)
                ingested.extend(result.get("ingested", []))
                if result.get("ingested") or result.get("mutations", {}).get("vault_writes"):
                    slugs_to_compile.add(slug)
                if self._heartbeat:
                    n = len(result.get("ingested", [])) or result.get("mutations", {}).get("vault_writes", 0)
                    await self._heartbeat.record_pulse("automation", f"Auto-ingested {n} files from inbox/")
            except Exception as exc:
                log.exception("Inbox watcher: inbox ingest failed")
                for fname in inbox_new:
                    fpath = str(self._resolve("inbox") / fname)
                    self._fail_counts[fpath] = self._fail_counts.get(fpath, 0) + 1
                if self._heartbeat:
                    await self._heartbeat.record_pulse("error", f"Inbox ingest failed: {exc}")

        # Process root folder files (user's content — COPY to KB inbox via global inbox)
        # Single KB → route there.  Multiple KBs → default to "general".
        if root_new:
            log.info("Inbox watcher: %d new files in root folder (queued in inbox/)", len(root_new))
            try:
                global_inbox = self._resolve("inbox")
                global_inbox.mkdir(parents=True, exist_ok=True)
                copied = 0
                for src in root_new:
                    dest = global_inbox / src.name
                    # Always copy — overwrites stale version for modified files
                    shutil.copy2(str(src), str(dest))
                    copied += 1

                    # Copy sibling images referenced by markdown files so
                    # the ingest pipeline can resolve ![alt](image.png) and
                    # ![[Pasted image.png]] refs.  Without this, images
                    # clipped alongside .md files (e.g., by Obsidian Clipper)
                    # sit unreferenced in the source dir while their markdown
                    # arrives in the inbox with dead image links.
                    if src.suffix.lower() in (".md", ".txt", ".html"):
                        self._copy_sibling_images(src, global_inbox)

                if copied:
                    # Route to target KB: single KB → that one, multiple → "general"
                    kbs = self._kb.list_kbs()
                    if len(kbs) == 1:
                        slug = kbs[0]["slug"]
                    else:
                        slug = "general"
                        if not any(k["slug"] == slug for k in kbs):
                            self._kb.create_kb("General")

                    kb_inbox = self._resolve(f"knowledge/{slug}/inbox")
                    kb_inbox.mkdir(parents=True, exist_ok=True)
                    for src in root_new:
                        gi_file = global_inbox / src.name
                        if gi_file.exists():
                            dest = kb_inbox / src.name
                            if dest.exists():
                                dest.unlink()  # remove stale version
                            shutil.move(str(gi_file), str(dest))
                    # Route through Agent.invoke_skill when available
                    # (Rule #17). Direct fallback on legacy / minimal installs.
                    result = await self._ingest_via_agent(slug)
                    ingested.extend(result.get("ingested", []))
                    if result.get("ingested") or result.get("mutations", {}).get("vault_writes"):
                        slugs_to_compile.add(slug)
                    if self._heartbeat:
                        n = len(result.get("ingested", [])) or result.get("mutations", {}).get("vault_writes", 0)
                        await self._heartbeat.record_pulse("automation", f"Auto-ingested {n} root files into KB {slug}")
            except Exception as exc:
                log.exception("Inbox watcher: root folder processing failed")
                for src in root_new:
                    self._fail_counts[str(src)] = self._fail_counts.get(str(src), 0) + 1
                if self._heartbeat:
                    await self._heartbeat.record_pulse("error", f"Root folder processing failed: {exc}")

        # Process per-KB inbox/ files
        for slug, files in kb_new.items():
            log.info("Inbox watcher: %d new files in knowledge/%s/inbox/: %s", len(files), slug, ", ".join(files[:5]))
            try:
                result = await self._ingest_via_agent(slug)
                ingested.extend(result.get("ingested", []))
                if result.get("ingested") or result.get("mutations", {}).get("vault_writes"):
                    slugs_to_compile.add(slug)
                if self._heartbeat:
                    await self._heartbeat.record_pulse("automation", f"Auto-ingested {len(files)} files into KB {slug}")
            except Exception:
                log.exception("Inbox watcher: KB %s ingest failed", slug)
                for fname in files:
                    fpath = str(self._resolve(f"knowledge/{slug}/inbox") / fname)
                    self._fail_counts[fpath] = self._fail_counts.get(fpath, 0) + 1

        # Compile once per KB after all ingests are done.
        # Bound with a timeout — local models can hang indefinitely.
        for slug in slugs_to_compile:
            try:
                await asyncio.wait_for(self._kb.compile_wiki(slug), timeout=300)
            except TimeoutError:
                log.warning("Inbox watcher: compile_wiki timed out for KB %s after 5min", slug)
            except Exception:
                log.exception("Inbox watcher: compile_wiki failed for KB %s", slug)

        # Note: post-ingest reflection via Editor was removed — it relied on
        # LLM delegation that doesn't actually invoke tools (small models
        # hallucinate). Reflection now happens explicitly via /api/kb/{slug}/reflect.

        if ingested:
            log.info("Inbox watcher: auto-ingested %d files total", len(ingested))

        # Mark processed files as known — but ONLY if they actually left the
        # inbox (moved to library) or were successfully ingested.  Files that
        # are still sitting in an inbox/ dir after a failed ingest should stay
        # "unknown" so they're retried on the next cycle or after restart.
        successfully_known = set()
        for fpath in current_files - deferred:
            p = Path(fpath)
            # If the file is still in an inbox/ directory AND no ingested
            # result references it, treat it as un-processed so it retries.
            if "/inbox/" in fpath.replace("\\", "/") and p.exists():
                fname = p.name
                if fname in [Path(i).name for i in ingested]:
                    successfully_known.add(fpath)
                elif fpath in self._known_files:
                    # Was already known from a prior cycle — check if it has
                    # a matching source page in the library (evidence of prior
                    # successful ingest).
                    kb_prefix = str(self._resolve("knowledge"))
                    if fpath.startswith(kb_prefix):
                        rel = p.relative_to(self._resolve("knowledge"))
                        slug = str(rel.parts[0])
                        lib_dir = self._resolve(f"knowledge/{slug}/library")
                        if lib_dir.is_dir() and any(
                            lf.stem == p.stem or lf.name == fname
                            for lf in lib_dir.rglob("*") if lf.is_file()
                        ):
                            successfully_known.add(fpath)  # genuinely processed before
                        else:
                            log.info("Inbox watcher: %s still in inbox with no library match — will retry", fname)
                    else:
                        successfully_known.add(fpath)
                else:
                    log.info("Inbox watcher: %s still in inbox after ingest — will retry next cycle", fname)
            else:
                successfully_known.add(fpath)

        self._known_files = successfully_known
        self._snapshot_file_stats(successfully_known)
        self._save_state()

        # Record cycle stats for observability
        import time
        self._last_cycle_at = time.time()
        self._last_cycle_ingested = len(ingested)
        self._last_cycle_deferred = len(deferred)
        self._cycle_count += 1

        # ── Detect deletions, renames, and moves in root folder ──
        if self._root_dir and self._root_dir != self._vault_dir:
            await self._detect_root_changes()

        return ingested

    async def _detect_root_changes(self) -> None:
        """Detect deleted, renamed, or moved files in the root folder.

        Uses content hashing (like git rename detection):
        - File gone from known path → search all files for same hash
        - Hash found at new path → renamed/moved → update source reference
        - Hash not found anywhere → truly deleted → mark wiki article as stale
        """
        import hashlib

        # Build current state: path → hash for all root files
        current_hashes: dict[str, str] = {}
        for f in self._root_dir.rglob("*"):
            if not f.is_file():
                continue
            rel_parts = f.relative_to(self._root_dir).parts
            if any(part in self._SKIP_DIRS or part.startswith(".") for part in rel_parts):
                continue
            if f.suffix.lower() in (".md", ".txt", ".html", ".pdf", ".png", ".jpg", ".jpeg", ".csv"):
                try:
                    content_hash = hashlib.md5(f.read_bytes()[:5000]).hexdigest()[:12]
                    current_hashes[str(f)] = content_hash
                except Exception:
                    pass

        # First run — just record state, don't detect changes
        if not self._root_file_hashes:
            self._root_file_hashes = current_hashes
            return

        # Find disappeared files
        old_paths = set(self._root_file_hashes.keys())
        new_paths = set(current_hashes.keys())
        disappeared = old_paths - new_paths

        if not disappeared:
            self._root_file_hashes = current_hashes
            return

        # Build reverse lookup: hash → current path
        hash_to_path: dict[str, str] = {}
        for path, h in current_hashes.items():
            hash_to_path[h] = path

        # Check each disappeared file
        for old_path in disappeared:
            old_hash = self._root_file_hashes[old_path]
            old_name = Path(old_path).name

            if old_hash in hash_to_path:
                # Found at a different path — renamed or moved
                new_path = hash_to_path[old_hash]
                new_name = Path(new_path).name
                log.info("Root file moved/renamed: %s → %s", old_name, new_name)
                # Update source reference in wiki (best effort)
                try:
                    self._update_source_reference(old_name, new_name, new_path)
                except Exception:
                    log.debug("Failed to update source reference for %s", old_name)
            else:
                # Hash not found anywhere — truly deleted
                log.info("Root file deleted: %s — marking wiki article as stale", old_name)
                try:
                    self._mark_article_stale(old_name)
                except Exception:
                    log.debug("Failed to mark article stale for %s", old_name)

        self._root_file_hashes = current_hashes

    def _mark_article_stale(self, filename: str) -> None:
        """Mark wiki source article as stale when source file is deleted."""
        import re
        slug = Path(filename).stem.lower().replace(" ", "-")

        # Search across all KBs for matching source page
        kb_dir = self._resolve("knowledge")
        if not kb_dir.is_dir():
            return

        for kb in kb_dir.iterdir():
            if not kb.is_dir():
                continue
            source_page = kb / "wiki" / "sources" / f"{slug}.md"
            if source_page.exists():
                content = source_page.read_text(encoding="utf-8")
                # Update status to stale
                if "status: verified" in content or "status: draft" in content:
                    content = re.sub(
                        r"status:\s*\w+",
                        "status: stale",
                        content, count=1,
                    )
                    # Add deletion note
                    content += (
                        f"\n\n> **Source deleted**: The original file `{filename}` "
                        f"was removed from the vault. This article may be outdated.\n"
                    )
                    source_page.write_text(content, encoding="utf-8")
                    log.info("Marked as stale: %s in KB %s", slug, kb.name)

    def _update_source_reference(self, old_name: str, new_name: str, new_path: str) -> None:
        """Update wiki source references when a file is renamed or moved."""
        kb_dir = self._resolve("knowledge")
        if not kb_dir.is_dir():
            return

        old_slug = Path(old_name).stem.lower().replace(" ", "-")
        new_slug = Path(new_name).stem.lower().replace(" ", "-")

        for kb in kb_dir.iterdir():
            if not kb.is_dir():
                continue
            source_page = kb / "wiki" / "sources" / f"{old_slug}.md"
            if source_page.exists():
                content = source_page.read_text(encoding="utf-8")
                # Update filename references
                content = content.replace(old_name, new_name)
                # Rename the file if slug changed
                if old_slug != new_slug:
                    new_page = kb / "wiki" / "sources" / f"{new_slug}.md"
                    source_page.rename(new_page)
                    new_page.write_text(content, encoding="utf-8")
                    log.info("Renamed source page: %s → %s in KB %s", old_slug, new_slug, kb.name)
                else:
                    source_page.write_text(content, encoding="utf-8")

    async def _delegate_to_librarian(self, task: str) -> str | None:
        """Try to delegate KB work to the Librarian subagent. Returns result or None."""
        if not self._orchestrator:
            return None
        try:
            result = await self._orchestrator.dispatch(
                agent_name="librarian",
                task=task,
                context="Triggered by inbox watcher automation.",
            )
            log.info("Librarian completed: %s", result.content[:100])
            return result.content
        except Exception:
            log.debug("Librarian delegation failed, falling back to direct KB call", exc_info=True)
            return None

    async def _delegate_to_editor(self, task: str) -> str | None:
        """Try to delegate quality/reflection work to the Editor subagent."""
        if not self._orchestrator:
            return None
        try:
            result = await self._orchestrator.dispatch(
                agent_name="editor",
                task=task,
                context="Triggered by post-compile reflection.",
            )
            log.info("Editor completed: %s", result.content[:100])
            return result.content
        except Exception:
            log.debug("Editor delegation failed", exc_info=True)
            return None

    async def _loop(self) -> None:
        while self._running:
            try:
                await self.check_and_ingest()
                self._last_error = ""
            except asyncio.CancelledError:
                break
            except Exception as exc:
                self._last_error = f"{type(exc).__name__}: {exc}"
                log.exception("Inbox watcher error")

            # Event-driven wait: wake immediately on a filesystem event,
            # otherwise fall through to the poll interval as a safety net.
            # If watchdog isn't running, _event_drain is never set and this
            # behaves exactly like the old `asyncio.sleep(interval)`.
            try:
                import time
                await asyncio.wait_for(
                    self._event_drain.wait(), timeout=self._interval,
                )
                # Woken by an event — record and clear for next cycle.
                # Small debounce so a burst of events produces one cycle.
                self._last_wake_at = time.time()
                self._wake_count += 1
                self._event_drain.clear()
                await asyncio.sleep(1.0)
            except TimeoutError:
                # Poll interval elapsed — normal tick, no events.
                pass
            except asyncio.CancelledError:
                break
