"""LLM status aggregator — one endpoint for the dashboard panel.

Combines live state (provider health, active model, pending work, last error)
with an error-pattern diagnosis so the UI can show actionable guidance instead
of raw tracebacks. Also exposes a manual "reconcile local tiers" action.
"""

from __future__ import annotations

import asyncio
import logging
import re
import subprocess
import time
from typing import Any

from fastapi import APIRouter, HTTPException, Request

log = logging.getLogger(__name__)

router = APIRouter(prefix="/llm", tags=["llm"])

# ── Probe cache ──────────────────────────────────────────────────────────
# /api/llm/status is polled every 15s by the dashboard. The probes inside
# (nvidia-smi, ollama ps, ollama /api/tags, ModelRouter YAML load) are
# blocking subprocess + sync httpx calls. Even when offloaded via
# asyncio.to_thread (so the event loop is free), each call still costs
# 50-300ms wall time and a Windows console flash. Cache the results for
# a short window so 4-second-apart polls share one probe round.
_PROBE_CACHE: dict[str, Any] = {
    "ts": 0.0,
    "gpu": None,
    "installed_models": [],
    "ollama_online": False,
    "tier_map": {},
}
_PROBE_TTL = 5.0  # seconds — short enough to feel live, long enough to absorb dashboard polling

# /api/llm/local-stack has its own cache because it does MORE work per call
# (Ollama /api/tags, /api/ps, models.yaml read, cost_log SQL aggregate,
# PowerShell driver probe). Worst-case per-call: 16+ seconds. Polled every
# 10s by the Local Models panel.
_LOCAL_STACK_CACHE: dict[str, Any] = {"ts": 0.0, "data": None}
# 60-second TTL: the dashboard's Local Models panel polls this every
# ~10s. Prior 8s TTL meant every poll missed the cache and triggered a
# fresh Ollama+driver probe (~4.5s blocking work via to_thread). Local
# model state genuinely doesn't change between consecutive 10s polls —
# you don't pull or load new models that fast. 60s strikes the balance:
# user perceives it as live (model count + loaded state refresh within
# a minute of changing), but burst polling shares one probe.
_LOCAL_STACK_TTL = 60.0

# Driver health is essentially immutable on the timescale of a session
# (you don't update GPU drivers between dashboard polls). Cache it MUCH
# longer to absorb any PowerShell launch hiccup.
_DRIVER_CACHE: dict[str, Any] = {"ts": 0.0, "data": None}
_DRIVER_TTL = 300.0  # 5 minutes


# ── Quiet subprocess on Windows ──────────────────────────────────────────
# Without CREATE_NO_WINDOW the subprocess flashes a console window each call.
# When the parent is pythonw.exe (no inherited console) every nvidia-smi /
# powershell / ollama ps call pops a visible cmd window for 50-200ms — which
# is exactly the "shell pulses every 2s" symptom the user reported once
# the LLM-status + local-stack panels started polling these helpers regularly.
# Always pass ``**_no_window_kwargs()`` to subprocess.run / Popen on background
# poll paths.

def _no_window_kwargs() -> dict[str, Any]:
    import sys
    if sys.platform != "win32":
        return {}
    return {"creationflags": 0x08000000}  # CREATE_NO_WINDOW


# ── GPU / VRAM detection ─────────────────────────────────────────────────
# Best-effort: try nvidia-smi first, then fall back to Ollama's ps output.
# Returns None when no GPU is detected. Never raises.

def _detect_gpu() -> dict[str, Any] | None:
    """Detect GPU name and VRAM via nvidia-smi or ollama ps."""
    # Try nvidia-smi (NVIDIA GPUs)
    try:
        out = subprocess.run(
            ["nvidia-smi", "--query-gpu=name,memory.total,memory.used,memory.free",
             "--format=csv,noheader,nounits"],
            capture_output=True, text=True, timeout=5,
            **_no_window_kwargs(),
        )
        if out.returncode == 0 and out.stdout.strip():
            gpus = []
            for line in out.stdout.strip().splitlines():
                parts = [p.strip() for p in line.split(",")]
                if len(parts) >= 4:
                    gpus.append({
                        "name": parts[0],
                        "vram_total_mb": int(float(parts[1])),
                        "vram_used_mb": int(float(parts[2])),
                        "vram_free_mb": int(float(parts[3])),
                    })
            if gpus:
                return {"source": "nvidia-smi", "gpus": gpus}
    except (FileNotFoundError, subprocess.TimeoutExpired, Exception):
        pass

    # Fallback: check ollama ps for running model VRAM usage
    try:
        out = subprocess.run(
            ["ollama", "ps"],
            capture_output=True, text=True, timeout=5,
            **_no_window_kwargs(),
        )
        if out.returncode == 0 and out.stdout.strip():
            lines = out.stdout.strip().splitlines()
            if len(lines) > 1:  # header + at least one model
                return {"source": "ollama-ps", "raw": out.stdout.strip()}
    except (FileNotFoundError, subprocess.TimeoutExpired, Exception):
        pass

    return None


# ── Error pattern → diagnosis ──────────────────────────────────────────────
# Each pattern returns: code (stable id), headline, advice, action (optional).
# Actions map to repair buttons the UI can wire up.
_PATTERNS: list[dict[str, Any]] = [
    {
        "match": re.compile(r"model not found|does not exist|not found in ollama", re.I),
        "code": "model_missing",
        "headline": "The configured model isn't installed.",
        "advice": (
            "Your tier routing points at a model Ollama doesn't have. "
            "Click 'Reconcile tiers' to remap tiers to your installed models, "
            "or pull the missing model manually with `ollama pull <name>`."
        ),
        "action": "reconcile_tiers",
    },
    {
        "match": re.compile(r"connection refused|connect.*refused|cannot connect|cannot reach|connection error|failed to connect", re.I),
        "code": "provider_unreachable",
        "headline": "Can't reach the LLM provider.",
        "advice": (
            "For local mode: make sure Ollama is running (`ollama serve` in a terminal, "
            "or launch the Ollama app). For cloud mode: check your internet connection."
        ),
        "action": "retry",
    },
    {
        "match": re.compile(r"invalid api key|incorrect api key|authentication.*fail|401|unauthorized", re.I),
        "code": "bad_credentials",
        "headline": "API key is invalid or missing.",
        "advice": "Open Settings → LLM and update the API key for your provider.",
        "action": "open_settings",
    },
    {
        "match": re.compile(r"rate.?limit|too many requests|429", re.I),
        "code": "rate_limited",
        "headline": "Provider is rate-limiting requests.",
        "advice": (
            "Wait a few minutes for the limit to reset, or switch to a different "
            "provider/model in Settings. Queued work will retry automatically."
        ),
        "action": "retry_all",
    },
    {
        "match": re.compile(r"quota|insufficient.*credit|billing|payment required|402", re.I),
        "code": "quota_exceeded",
        "headline": "Out of credits / quota exceeded.",
        "advice": "Top up your provider account or switch to local mode in Settings → LLM.",
        "action": "open_settings",
    },
    {
        "match": re.compile(r"context.?length|context.?window|token.*limit|too.*long", re.I),
        "code": "context_too_long",
        "headline": "Input too large for this model's context window.",
        "advice": "Switch to a model with a larger context window, or split the task into smaller chunks.",
        "action": "open_settings",
    },
    {
        "match": re.compile(r"timeout|timed out", re.I),
        "code": "timeout",
        "headline": "LLM call timed out.",
        "advice": (
            "For local mode: the model may be too large for your hardware — try a smaller model. "
            "For cloud: transient network issue, retry usually works."
        ),
        "action": "retry",
    },
]


def _diagnose(error: str) -> dict[str, Any] | None:
    """Match an error message against known patterns. Returns None if no match."""
    if not error:
        return None
    for pat in _PATTERNS:
        if pat["match"].search(error):
            return {
                "code": pat["code"],
                "headline": pat["headline"],
                "advice": pat["advice"],
                "action": pat.get("action"),
            }
    return {
        "code": "unknown",
        "headline": "LLM error (unclassified).",
        "advice": "Check the full error in /logs. If it recurs, check provider status and Settings → LLM.",
        "action": None,
    }


def _run_blocking_probes(mode: str, provider: str, settings_active_model: str, settings_skills_dir: Any, settings_quality_profile: str) -> dict[str, Any]:
    """Run the blocking probes (subprocess, sync httpx, YAML load) in one thread.

    Returns ``{"gpu", "installed_models", "ollama_online", "tier_map"}``.
    Always succeeds — every sub-probe is independently try/except'd.
    Called via ``asyncio.to_thread`` so the event loop stays free.
    """
    # Tier map — cheap (YAML load) but still blocking I/O.
    tier_map: dict[str, str] = {}
    try:
        from agntspace.llm.model_router import ModelRouter
        mr = ModelRouter(
            provider=provider,
            profile="local" if mode == "local" else settings_quality_profile,
            default_model=settings_active_model,
            skills_dir=settings_skills_dir,
            mode=mode,
        )
        tier_map = mr.get_profile_summary() or {}
    except Exception:
        pass

    # Installed Ollama models (sync httpx.get inside _installed_models).
    installed_models: list[str] = []
    ollama_online = False
    if mode == "local":
        try:
            from agntspace.llm.ollama_discovery import _installed_models
            installed = _installed_models()
            installed_models = [m["name"] for m in installed]
            ollama_online = bool(installed_models)
        except Exception:
            pass

    # GPU detection (subprocess.run × up to 2).
    gpu_info: dict[str, Any] | None = None
    if mode == "local":
        gpu_info = _detect_gpu()

    return {
        "gpu": gpu_info,
        "installed_models": installed_models,
        "ollama_online": ollama_online,
        "tier_map": tier_map,
    }


@router.get("/status")
async def llm_status(request: Request) -> dict:
    """Aggregated LLM status for the dashboard panel."""
    from agntspace.infra.config import get_settings
    settings = get_settings()

    # LLM instance isn't cached on app.state — pull live state from the
    # agent's provider and the work queue (populated by heartbeat pings).
    agent = getattr(request.app.state, "agent", None)
    llm = getattr(agent, "_llm", None) if agent else None
    wq = getattr(request.app.state, "work_queue", None)
    error_logger = getattr(request.app.state, "error_logger", None)

    mode = settings.llm.mode
    provider = settings.llm.local_provider if mode == "local" else settings.llm.cloud_provider

    active_model = "unknown"
    last_error = ""
    provider_healthy = True
    if llm:
        active_model = getattr(llm, "model_name", "") or getattr(llm, "_model", "") or active_model
        last_error = getattr(llm, "last_error", "") or ""
        provider_healthy = bool(getattr(llm, "healthy", True))

    pending_jobs = 0
    wq_healthy = True
    if wq:
        wq_healthy = bool(getattr(wq, "llm_healthy", True))
        # Heartbeat writes the active model into the work queue; prefer it
        # when we don't have a better source.
        wq_model = getattr(wq, "active_model", "") or ""
        if (not active_model or active_model == "unknown") and wq_model:
            active_model = wq_model
        try:
            status = await wq.get_status()
            pending_jobs = int(status.get("total_pending", 0))
        except Exception:
            pass

    # Recent LLM-ish errors from the error log (non-blocking)
    recent_errors: list[dict[str, Any]] = []
    if error_logger:
        try:
            rows = await error_logger.get_errors(
                limit=5,
                source="llm",
                resolved=False,
            )
            recent_errors = [
                {
                    "id": r["id"],
                    "timestamp": r["timestamp"],
                    "level": r["level"],
                    "source": r["source"],
                    "message": r["message"],
                }
                for r in rows
            ]
        except Exception:
            pass

    # Fall back to the freshest recent error message if llm.last_error is empty
    if not last_error and recent_errors:
        last_error = recent_errors[0]["message"]

    # Tier map / installed models / GPU — all blocking I/O.
    # 1) Prefer the cached ModelRouter on app.state when available — avoids
    #    YAML reloads on every dashboard poll.
    # 2) For everything else, run the blocking probes off-loop via
    #    asyncio.to_thread, with a short cache so 15s-poll bursts share one round.
    tier_map: dict[str, str] = {}
    cached_router = getattr(request.app.state, "model_router", None)
    if cached_router is not None:
        try:
            tier_map = cached_router.get_profile_summary() or {}
        except Exception:
            tier_map = {}

    now = time.monotonic()
    if (now - _PROBE_CACHE["ts"]) < _PROBE_TTL and _PROBE_CACHE["ts"] > 0:
        probe = {
            "gpu": _PROBE_CACHE["gpu"],
            "installed_models": _PROBE_CACHE["installed_models"],
            "ollama_online": _PROBE_CACHE["ollama_online"],
            "tier_map": _PROBE_CACHE["tier_map"],
        }
    else:
        try:
            probe = await asyncio.to_thread(
                _run_blocking_probes,
                mode,
                provider,
                settings.active_model,
                settings.skills_dir,
                settings.llm.quality_profile,
            )
        except Exception:
            probe = {"gpu": None, "installed_models": [], "ollama_online": False, "tier_map": {}}
        _PROBE_CACHE["ts"] = now
        _PROBE_CACHE["gpu"] = probe["gpu"]
        _PROBE_CACHE["installed_models"] = probe["installed_models"]
        _PROBE_CACHE["ollama_online"] = probe["ollama_online"]
        _PROBE_CACHE["tier_map"] = probe["tier_map"]

    # Prefer the cached-router tier_map (always live); fall back to probe value.
    if not tier_map:
        tier_map = probe["tier_map"]
    installed_models: list[str] = probe["installed_models"]
    ollama_online: bool = probe["ollama_online"]

    healthy = provider_healthy and wq_healthy and not last_error

    # In local mode, suppress stale errors when Ollama is actually online now.
    diagnosis = _diagnose(last_error) if not healthy else None
    if diagnosis and mode == "local" and ollama_online:
        stale = False
        if diagnosis["code"] == "model_missing":
            # Model might have been pulled since the error was recorded.
            bare_model = active_model.split("/", 1)[-1] if "/" in active_model else active_model
            if bare_model in installed_models:
                stale = True
        elif diagnosis["code"] == "provider_unreachable":
            # Ollama is online now — the "cannot reach" error is stale.
            stale = True
        if stale:
            healthy = True
            diagnosis = None
            last_error = ""
            if llm:
                llm._healthy = True  # noqa: SLF001
                llm._last_error = ""  # noqa: SLF001

    # GPU/VRAM detection — already done in the off-loop probe above.
    gpu_info: dict[str, Any] | None = probe["gpu"]

    return {
        "mode": mode,
        "provider": provider,
        "active_model": active_model,
        "healthy": healthy,
        "provider_healthy": provider_healthy,
        "queue_healthy": wq_healthy,
        "last_error": last_error,
        "pending_jobs": pending_jobs,
        "tier_map": tier_map,
        "installed_models": installed_models,
        "ollama_online": ollama_online if mode == "local" else None,
        "recent_errors": recent_errors,
        "diagnosis": diagnosis,
        "gpu": gpu_info,
    }


@router.post("/reconcile-tiers")
async def reconcile_tiers(request: Request) -> dict:
    """Re-run local tier reconciliation against the live Ollama instance.

    Rewrites `models.yaml` if any tier points to a missing model. Reloads the
    live `ModelRouter` so the change takes effect without a restart.
    """
    from agntspace.infra.config import get_settings
    settings = get_settings()

    if settings.llm.mode != "local":
        raise HTTPException(400, "Tier reconciliation only applies in local mode")

    from agntspace.llm.ollama_discovery import reconcile_local_tiers
    result = reconcile_local_tiers(settings.skills_dir)

    # Reload the running ModelRouter so background services see the new map
    mr = getattr(request.app.state, "model_router", None)
    if mr is not None and result.get("status") in ("updated", "unchanged"):
        try:
            mr._load(settings.skills_dir)  # noqa: SLF001
            result["router_reloaded"] = True
        except Exception as exc:
            log.warning("ModelRouter reload failed: %s", exc)
            result["router_reloaded"] = False

    # Clear stale provider health state — the old "model not found" error
    # is no longer relevant after reconciliation remapped the tiers.
    agent = getattr(request.app.state, "agent", None)
    llm = getattr(agent, "_llm", None) if agent else None
    if llm and result.get("status") in ("updated", "unchanged"):
        llm._healthy = True  # noqa: SLF001
        llm._last_error = ""  # noqa: SLF001
        result["provider_health_cleared"] = True

    return result


# ── Local-stack dashboard ────────────────────────────────────────────────
#
# Gives the UI a single aggregated view of "what local LLM infrastructure
# do I have, what's active right now, what's each model being used for."
# Intentionally always-available regardless of cloud/local mode, because
# even cloud-mode users run Ollama for bge-m3 embeddings and deserve to
# see that stack rather than have it feel invisible.

def _ollama_ps() -> list[dict[str, Any]]:
    """Query Ollama's /api/ps for currently-loaded models.

    Returns [] on any error. Each entry has ``name``, ``size_vram`` (bytes),
    ``expires_at`` (ISO string when the model will unload if idle).
    """
    try:
        import httpx
        with httpx.Client(timeout=3.0) as client:
            r = client.get("http://localhost:11434/api/ps")
            if r.status_code != 200:
                return []
            data = r.json()
            return [
                {
                    "name": m.get("name", ""),
                    "size_vram": m.get("size_vram", 0),
                    "expires_at": m.get("expires_at", ""),
                }
                for m in data.get("models", [])
            ]
    except Exception:
        return []


def _ollama_tags() -> list[dict[str, Any]]:
    """Query Ollama /api/tags for installed models. [] on error."""
    try:
        import httpx
        with httpx.Client(timeout=3.0) as client:
            r = client.get("http://localhost:11434/api/tags")
            if r.status_code != 200:
                return []
            data = r.json()
            return list(data.get("models", []))
    except Exception:
        return []


def _tier_assignments(skills_dir) -> dict[str, str]:
    """Read local.ollama tier -> model map from models.yaml.

    Returns ``{tier_name: model_name}``. Empty dict on any error, so callers
    just render "unassigned" for every model instead of breaking.
    """
    try:
        import yaml
        p = skills_dir / "_meta" / "model-routing" / "config" / "models.yaml"
        if not p.is_file():
            return {}
        doc = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
        return dict(doc.get("profiles", {}).get("local", {}).get("ollama", {}))
    except Exception:
        return {}


async def _usage_per_model(db) -> dict[str, dict[str, Any]]:
    """Aggregate cost_log by model: call count, last-used, tokens.

    Returns ``{model_name: {calls, last_used, input_tokens, output_tokens}}``.
    Empty on any error — usage is nice-to-have, not load-bearing.
    """
    if db is None:
        return {}
    try:
        q = (
            "SELECT model, COUNT(*) AS calls, MAX(ts) AS last_used, "
            "  SUM(input_tokens) AS in_tokens, SUM(output_tokens) AS out_tokens "
            "FROM cost_log GROUP BY model"
        )
        async with db.execute(q) as cursor:
            rows = await cursor.fetchall()
        out: dict[str, dict[str, Any]] = {}
        for row in rows:
            # rows may be tuples OR sqlite.Row — index works on both
            model = row[0] or ""
            if not model:
                continue
            out[model] = {
                "calls": int(row[1] or 0),
                "last_used": row[2] or "",
                "input_tokens": int(row[3] or 0),
                "output_tokens": int(row[4] or 0),
            }
        return out
    except Exception:
        return {}


def _embedding_activity() -> dict[str, Any]:
    """Peek at OllamaEmbedModel's per-process activity counters.

    The singleton EmbeddingsService maintains a lightweight in-memory record
    of recent calls so the UI can render the "⚡ Active" pulse even though
    embedding traffic doesn't currently flow through cost_log. Returns an
    empty dict when the service isn't wired or has no activity yet.
    """
    try:
        from agntspace.vault import embeddings as _emb
        return {
            "total_calls": getattr(_emb, "_total_embed_calls", 0),
            "last_call_iso": getattr(_emb, "_last_embed_call_iso", ""),
            "last_model": getattr(_emb, "_last_embed_model", ""),
        }
    except Exception:
        return {}


def _driver_health() -> dict[str, Any] | None:
    """Best-effort GPU driver staleness check (Windows only).

    Returns ``{name, driver_version, driver_date, stale_days, warning}``
    or ``None`` when we can't introspect (non-Windows, no GPU, failure).
    """
    import sys
    if sys.platform != "win32":
        return None
    try:
        import subprocess
        result = subprocess.run(
            ["powershell", "-NoProfile", "-NonInteractive", "-Command",
             "Get-CimInstance Win32_VideoController | "
             "Select-Object Name, DriverVersion, DriverDate | "
             "ConvertTo-Json -Compress"],
            capture_output=True, text=True, timeout=10,
            **_no_window_kwargs(),
        )
        if result.returncode != 0 or not result.stdout.strip():
            return None
        import json as _json
        from datetime import datetime
        data = _json.loads(result.stdout)
        if isinstance(data, dict):
            data = [data]
        gpus: list[dict[str, Any]] = []
        now = datetime.now()
        for gpu in data:
            name = gpu.get("Name", "")
            version = gpu.get("DriverVersion", "")
            raw_date = gpu.get("DriverDate", "")
            stale_days: int | None = None
            try:
                # PowerShell returns "/Date(epoch_ms)/"
                import re as _re
                m = _re.search(r"(\d{10,13})", str(raw_date))
                if m:
                    ms = int(m.group(1))
                    dt = datetime.fromtimestamp(ms / 1000)
                    stale_days = (now - dt).days
            except Exception:
                pass
            warn = ""
            if stale_days is not None and stale_days > 90:
                warn = f"Driver is {stale_days} days old — update recommended"
            gpus.append({
                "name": name,
                "driver_version": version,
                "stale_days": stale_days,
                "warning": warn,
            })
        return {"gpus": gpus}
    except Exception:
        return None


def _build_local_stack_blocking(skills_dir) -> dict[str, Any]:
    """Pure sync helper — runs every blocking probe in one thread.

    Returns ``{tags, loaded, tier_map, embed_model, driver}`` so the caller
    can do the cheap dict-shaping work back on the loop without waiting
    for any subprocess or sync-httpx round trip.
    """
    tags = _ollama_tags()
    loaded = _ollama_ps()
    tier_map = _tier_assignments(skills_dir)
    try:
        from agntspace.vault.embeddings import DEFAULT_EMBED_MODEL
        embed_model = DEFAULT_EMBED_MODEL
    except Exception:
        embed_model = "bge-m3"

    # Driver health: cache 5 min — drivers don't change between dashboard polls.
    now = time.monotonic()
    if (now - _DRIVER_CACHE["ts"]) < _DRIVER_TTL and _DRIVER_CACHE["data"] is not None:
        driver = _DRIVER_CACHE["data"]
    else:
        driver = _driver_health()
        _DRIVER_CACHE["ts"] = now
        _DRIVER_CACHE["data"] = driver

    return {
        "tags": tags,
        "loaded": loaded,
        "tier_map": tier_map,
        "embed_model": embed_model,
        "driver": driver,
    }


@router.get("/local-stack")
async def get_local_stack(request: Request) -> dict:
    """Comprehensive status of local LLM infrastructure.

    Returns installed models, tier assignments, VRAM status, usage stats,
    Ollama process info, and driver warnings. Always safe to call — degrades
    gracefully when Ollama is offline, the cost DB is missing, or the user
    is on a non-Windows platform.

    Blocking work (Ollama HTTP probes, models.yaml load, PowerShell driver
    query) runs off the asyncio loop via to_thread, AND the whole bundle
    is cached for 8s — every endpoint queueing behind this call previously
    saw 100+ second waits because each invocation took 5-15s and they
    serialised on the loop. With the cache + to_thread, a 10s-poll burst
    pays the probe cost ONCE.
    """
    from agntspace.infra.config import get_settings
    settings = get_settings()

    # 8-second probe cache: returns a cached snapshot when fresh, runs the
    # blocking probes off-loop when stale.
    now = time.monotonic()
    if (now - _LOCAL_STACK_CACHE["ts"]) < _LOCAL_STACK_TTL and _LOCAL_STACK_CACHE["data"] is not None:
        probe = _LOCAL_STACK_CACHE["data"]
    else:
        try:
            probe = await asyncio.to_thread(_build_local_stack_blocking, settings.skills_dir)
        except Exception:
            probe = {"tags": [], "loaded": [], "tier_map": {}, "embed_model": "bge-m3", "driver": None}
        _LOCAL_STACK_CACHE["ts"] = now
        _LOCAL_STACK_CACHE["data"] = probe

    tags = probe["tags"]
    loaded = probe["loaded"]
    tier_map = probe["tier_map"]
    embed_model = probe["embed_model"]
    driver = probe["driver"]

    loaded_names = {m["name"] for m in loaded}
    loaded_by_name = {m["name"]: m for m in loaded}

    # Invert: model -> tier. A model can be in multiple tiers (e.g. same name
    # mapped to both "reasoning" and "capable"); keep them all.
    model_to_tiers: dict[str, list[str]] = {}
    for tier, name in tier_map.items():
        model_to_tiers.setdefault(name, []).append(tier)

    db = getattr(request.app.state, "db", None)
    usage = await _usage_per_model(db)

    # Embedding activity is tracked in-memory by OllamaEmbedModel — it does
    # NOT write to cost_log (would multiply row volume by 100x+ on heavy
    # ingest). Read it once and overlay onto whichever model IS the
    # embedding model so its row reports accurate "last used" + call count
    # instead of the cost_log default of "never". Without this overlay
    # bge-m3 looks idle while embeddings are firing constantly during
    # ingest / graph build / chat — the contradiction the LLM panel's
    # "Embeddings only" badge would otherwise expose.
    emb_activity = _embedding_activity()
    emb_last_call = emb_activity.get("last_call_iso", "") or ""
    emb_total_calls = int(emb_activity.get("total_calls", 0) or 0)

    # Build per-model records
    models_out = []
    for m in tags:
        name = m.get("name", "")
        bare = name.split(":")[0]
        # Match usage whether stored as bare or tagged
        u = usage.get(name) or usage.get(bare) or {}
        tiers = model_to_tiers.get(name) or model_to_tiers.get(bare, [])
        role: list[str] = list(tiers)
        is_embed = bare == embed_model or name.startswith(f"{embed_model}:")
        if is_embed:
            role.append("embeddings")
        if not role:
            role.append("unassigned")

        # Resolve last-used + call-count: cost_log is the source for chat,
        # embedding_activity is the source for embeddings. When a model
        # serves both (rare — same model wired to chat + embeddings) we
        # take the MAX timestamp + SUM of call counts so neither stream
        # gets erased.
        cost_last = u.get("last_used", "") or ""
        cost_calls = int(u.get("calls", 0) or 0)
        if is_embed and emb_last_call:
            last_used_iso = max(cost_last, emb_last_call) if cost_last else emb_last_call
            calls = cost_calls + emb_total_calls
        else:
            last_used_iso = cost_last
            calls = cost_calls

        details = m.get("details", {}) or {}
        loaded_info = loaded_by_name.get(name) or loaded_by_name.get(f"{bare}:latest")
        models_out.append({
            "name": name,
            "bare_name": bare,
            "size_bytes": m.get("size", 0),
            "parameter_size": details.get("parameter_size", ""),
            "quantization": details.get("quantization_level", ""),
            "family": details.get("family", ""),
            "tiers": tiers,
            "roles": role,
            "loaded": name in loaded_names,
            "vram_bytes": (loaded_info or {}).get("size_vram", 0),
            "expires_at": (loaded_info or {}).get("expires_at", ""),
            "calls": calls,
            "last_used_iso": last_used_iso,
            "input_tokens": u.get("input_tokens", 0),
            "output_tokens": u.get("output_tokens", 0),
        })
    # Sort: active/embeddings first, then by call count desc, then by size
    def _sort_key(m: dict) -> tuple:
        active = -1 if m["loaded"] else 0
        has_role = -1 if "unassigned" not in m["roles"] else 0
        return (active, has_role, -m["calls"], -m["size_bytes"])
    models_out.sort(key=_sort_key)

    # Ollama process info
    ollama_info: dict[str, Any] = {"running": bool(tags), "models_installed": len(tags)}

    total_disk_bytes = sum(m.get("size", 0) for m in tags)
    total_vram_bytes = sum((m.get("size_vram", 0) or 0) for m in loaded)

    # Hybrid routing snapshot — exposes which tiers (if any) are routed to
    # the local Ollama stack in cloud mode. Lets the UI show a "Routed"
    # badge + an explainer header so users understand WHY their qwen2.5
    # is going hot in cloud mode. Read directly from the loaded
    # ModelRouter so YAML edits propagate without restart.
    routing_block: dict[str, Any] = {
        "mode": settings.llm.mode,
        "cloud_use_local_for_tiers": [],
        "honor_prefer_local": False,
        "local_reachable": False,
    }
    try:
        mr = getattr(request.app.state, "model_router", None)
        if mr is not None:
            routing_block["cloud_use_local_for_tiers"] = sorted(mr._cloud_use_local_for_tiers)  # noqa: SLF001
            routing_block["honor_prefer_local"] = bool(mr._honor_prefer_local)  # noqa: SLF001
            # Cheap — uses the cached probe from ModelRouter (30s TTL).
            routing_block["local_reachable"] = bool(mr._local_reachable_cached())  # noqa: SLF001
    except Exception:
        # Routing visibility is nice-to-have. Never break the panel.
        pass

    return {
        "ollama": ollama_info,
        "models": models_out,
        "loaded_count": len(loaded),
        "total_count": len(tags),
        "total_disk_bytes": total_disk_bytes,
        "total_vram_bytes": total_vram_bytes,
        "embedding_model": embed_model,
        "embedding_activity": _embedding_activity(),
        "driver": driver,
        "routing": routing_block,
    }


@router.post("/local-stack/test/{model:path}")
async def test_local_model(request: Request, model: str) -> dict:
    """Fire a trivial prompt at a specific Ollama model and report round-trip.

    Returns ``{model, latency_ms, ok, error?, response}``. Warms up the
    model as a side effect — the NEXT call from AgntSpace will be fast
    because the model is now in VRAM.

    Times out after 60s (cold bge-m3 can take ~40s on first load).
    """
    import time
    try:
        import httpx
    except ImportError as exc:
        raise HTTPException(500, "httpx not installed") from exc

    payload = {
        "model": model,
        "prompt": "Say 'ok' in one word.",
        "stream": False,
        "options": {"num_predict": 8, "temperature": 0.0},
    }
    started = time.perf_counter()
    try:
        async with httpx.AsyncClient(timeout=60.0) as client:
            r = await client.post("http://localhost:11434/api/generate", json=payload)
    except httpx.TimeoutException:
        return {
            "model": model,
            "ok": False,
            "error": "Timed out after 60s — GPU cold-start or driver issue",
            "latency_ms": int((time.perf_counter() - started) * 1000),
        }
    except httpx.ConnectError:
        return {
            "model": model,
            "ok": False,
            "error": "Ollama not reachable at localhost:11434 — is it running?",
            "latency_ms": 0,
        }
    except Exception as exc:
        return {
            "model": model,
            "ok": False,
            "error": f"Unexpected: {exc}",
            "latency_ms": int((time.perf_counter() - started) * 1000),
        }
    latency_ms = int((time.perf_counter() - started) * 1000)

    if r.status_code == 404:
        return {
            "model": model,
            "ok": False,
            "error": f"Model '{model}' not installed",
            "latency_ms": latency_ms,
        }
    if r.status_code != 200:
        return {
            "model": model,
            "ok": False,
            "error": f"Ollama returned HTTP {r.status_code}",
            "latency_ms": latency_ms,
        }
    try:
        data = r.json()
    except Exception:
        return {
            "model": model,
            "ok": False,
            "error": "Could not parse Ollama response",
            "latency_ms": latency_ms,
        }
    return {
        "model": model,
        "ok": True,
        "latency_ms": latency_ms,
        "response": (data.get("response") or "").strip()[:120],
        "eval_count": data.get("eval_count", 0),
        "load_duration_ms": int((data.get("load_duration", 0) or 0) / 1_000_000),
        "total_duration_ms": int((data.get("total_duration", 0) or 0) / 1_000_000),
    }
