"""Memory engine — generates daily memory files, compacts into layered summaries.

This is the brain's consolidation process:
1. Daily: gathers all activity → writes memory/YYYY/MM/memory_YYYY-MM-DD.md
2. Weekly: compacts dailies → memory/summaries/week.md
3. Monthly: compacts weeklies → memory/summaries/month.md
4. Quarterly: compacts monthlies → memory/summaries/quarter.md
5. Yearly: compacts quarterlies → memory/summaries/year.md
6. Long-term: distills permanent facts → memory/summaries/longterm.md

Each layer only reads from the layer below it. MEMORY.md is an assembled view.
Daily files are never deleted — they are the epistemic ground truth.
Also supports real-time fact extraction during conversations.
"""

from __future__ import annotations

import logging
import re
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from agntspace.coach.service import CoachService
    from agntspace.infra.cost_tracker import CostTracker
    from agntspace.integrations.email_client import EmailIntegration
    from agntspace.journal.service import JournalService
    from agntspace.llm.base import LLMProvider
    from agntspace.memory.conversation import ConversationMemory
    from agntspace.memory.graph import KnowledgeGraph
    from agntspace.tasks.service import TaskService
    from agntspace.trust.service import TrustService
    from agntspace.vault.reader import VaultReader
    from agntspace.vault.writer import VaultWriter

log = logging.getLogger(__name__)


class MemoryEngine:
    """Generates daily memory files and maintains the knowledge base."""

    def __init__(
        self,
        vault_reader: VaultReader,
        vault_writer: VaultWriter,
        llm: LLMProvider,
        conversation_memory: ConversationMemory | None = None,
        journal: JournalService | None = None,
        task_service: TaskService | None = None,
        trust_service: TrustService | None = None,
        coach: CoachService | None = None,
        email: EmailIntegration | None = None,
        cost_tracker: CostTracker | None = None,
        knowledge_graph: KnowledgeGraph | None = None,
        skill_loader: object | None = None,
        model_router: object | None = None,
    ) -> None:
        self._reader = vault_reader
        self._writer = vault_writer
        self._llm = llm
        self._model_router = model_router
        self._memory = conversation_memory
        self._journal = journal
        self._tasks = task_service
        self._trust = trust_service
        self._coach = coach
        self._email = email
        self._costs = cost_tracker
        self._graph = knowledge_graph
        self._skill_loader = skill_loader
        self._agent: object | None = None  # injected post-hoc from main.py
        self._orchestrator: object | None = None  # injected post-hoc for activity tracking
        self._activity: object | None = None  # injected post-hoc — ActivityLogger
        self._decisions: object | None = None  # injected post-hoc — DecisionLogger
        self._contract: object | None = None  # injected post-hoc — ContractEnforcer

    def _get_model_name(self) -> str:
        """Extract the current model name."""
        try:
            if self._agent and hasattr(self._agent, "_llm") and hasattr(self._agent._llm, "_model"):
                return self._agent._llm._model
        except Exception:
            pass
        try:
            if hasattr(self._llm, "_model"):
                return self._llm._model
        except Exception:
            pass
        return "unknown"

    async def generate_daily_memory(self, date: str | None = None) -> dict:
        """Generate the daily memory file for a given date via the main agent.

        Gathers all activity, sends to agent for synthesis with [[wiki-links]],
        writes memory file, updates MEMORY.md, rebuilds graph.

        Wrapped in ``scoped_operation`` (Rule #17 Tier B) so every vault
        write + triple extraction + compaction under this call shares
        one ``memory-consolidate-*`` correlation id visible in /decisions.
        """
        if not date:
            from agntspace.infra.timezone import local_today
            date = local_today()

        # Gather all activity for this day — no vault writes yet so this
        # runs OUTSIDE the scoped_operation block. The early-return path
        # for empty activity doesn't need observability wrapping either.
        activity = await self._gather_activity(date)

        if not activity.strip():
            return {"date": date, "status": "no_activity"}

        from agntspace.activity.scoped import scoped_operation

        year, month, day = date.split("-")
        memory_path = f"memory/{year}/{month}/memory_{date}.md"

        async with scoped_operation(
            writer=self._writer,
            contract=self._contract,
            decisions=self._decisions,
            activity=self._activity,
            actor="main",
            action_type="memory_consolidate",
            action_target=memory_path,
            reason="memory_consolidate",
            scope_prefix="memory-consolidate",
            rationale=f"Daily memory consolidation for {date}",
            caller="daily_cycle",
            skill_context=["memory-consolidate"],
        ) as _scope:
            return await self._do_generate_daily_memory(date, activity, memory_path, _scope)

    async def _do_generate_daily_memory(
        self, date: str, activity: str, memory_path: str, _scope: object,
    ) -> dict:
        """Internal memory generation body — caller owns the scoped_operation."""
        # (Original body below. Kept inside a helper so callers that
        # already own a scope — e.g., post-EOD cascade — don't re-wrap.)

        # Synthesis prompt: asks for prose + permanent facts ONLY.
        # Triples are NOT requested here — entity extraction runs as a
        # separate pass on the synthesized memory_content, so structural
        # labels in `activity` (e.g., "## User Journal") can never leak
        # into the graph as entities.
        task_instruction = (
            f"Synthesize today's activity ({date}) into a daily memory file.\n\n"
            "Create [[wiki-links]] for every person, company, project, and key concept.\n"
            "Link generously — more connections = smarter graph.\n\n"
            "CRITICAL: NEVER fabricate, invent, or hallucinate information. "
            "Every person, project, task, and fact you mention MUST appear in "
            "the activity data provided below. If the day was quiet with little "
            "activity, produce a short memory noting that — do NOT fill gaps "
            "with invented content.\n\n"
            "Output exactly two fenced blocks:\n"
            "1. ```memory\n(the daily memory content with sections and [[links]])\n```\n"
            "2. ```permanent\n(new facts for MEMORY.md, or 'no updates')\n```\n\n"
            "Output ONLY these two fenced blocks, nothing else."
        )

        # Track activity so constellation shows memory consolidation
        if self._orchestrator and hasattr(self._orchestrator, "_active_tasks"):
            self._orchestrator._active_tasks["main"] = f"Memory consolidation ({date})"
        try:
            response_text = await self._agent.process_task(task=task_instruction, context=activity, model_tier="capable")
        finally:
            if self._orchestrator and hasattr(self._orchestrator, "_active_tasks"):
                self._orchestrator._active_tasks.pop("main", None)

        # Sanitize — local models sometimes output raw JSON
        from agntspace.automation.daily_cycle import sanitize_llm_output
        response_text = sanitize_llm_output(response_text)

        # ── Fabrication detection (Rule #16) ──────────────────────
        try:
            from agntspace.automation.fabrication_detector import check_memory_fabrication
            fab_report = check_memory_fabrication(response_text, activity, layer="daily", vault_reader=self._reader)
            if fab_report.was_modified:
                response_text = fab_report.clean_output
                log.warning(
                    "Daily memory consolidation: stripped %d fabricated names: %s",
                    len(fab_report.fabricated_names),
                    ", ".join(fab_report.fabricated_names),
                )
        except Exception as fab_exc:
            log.debug("Fabrication check failed for daily memory: %s", fab_exc)

        # Parse output
        memory_content = self._extract_block(response_text, "memory")
        permanent_updates = self._extract_block(response_text, "permanent")

        if not memory_content:
            # Fallback: strip any fenced blocks from raw response to avoid
            # leaking ```permanent and ```triples sections into the memory file
            memory_content = self._strip_fenced_blocks(response_text)

        # Write daily memory file — memory_path was computed by the
        # outer scope wrapper and passed in.
        now = datetime.now(UTC)
        model = self._get_model_name()

        full_content = (
            f"---\n"
            f'title: "Memory — {date}"\n'
            f'date: "{date}"\n'
            f'generated_at: "{now.strftime("%H:%M UTC")}"\n'
            f"type: daily_memory\n"
            f"author: agent\n"
            f"agent: main\n"
            f'model: "{model}"\n'
            f'description: "Daily memory consolidation — synthesized from journal, tasks, goals, conversations, and observations. Ground truth for what happened this day."\n'
            f"---\n\n"
            f"{memory_content}"
        )
        self._writer.write(memory_path, full_content)
        log.info("Daily memory written: %s", memory_path)

        # Write permanent facts as PENDING (user must approve before they go into MEMORY.md)
        if permanent_updates and "no updates" not in permanent_updates.lower():
            self._write_pending_memory(permanent_updates, date)

        # Extract triples from the SYNTHESIZED memory content — not from the
        # raw activity. The synthesized content has been rewritten by the
        # agent as prose about real entities (with [[wikilinks]]) and no
        # longer carries the engine's own section labels ("## User Journal",
        # etc.), so extraction cannot mistake scaffolding for entities.
        # This separation is the architectural fix for label-leakage.
        triples_extracted = 0
        if self._graph and memory_content:
            extracted = await self._graph.extract_triples_llm(
                memory_content, source_file=memory_path, source_date=date, authority="system",
            )
            triples_extracted = len(extracted)

            # Rebuild wiki-link graph (also loads SPO edges)
            edges = self._graph.build()
            log.info("Knowledge graph rebuilt: %d edges, %d new triples", edges, triples_extracted)

        # Auto-compact the week layer after daily generation (natural cadence)
        try:
            if self._agent:
                await self._compact_layer("week")
                self._assemble_memory_md()
                log.info("Week summary auto-compacted after daily memory generation")
        except Exception:
            log.debug("Week auto-compaction failed", exc_info=True)

        return {
            "date": date,
            "path": memory_path,
            "permanent_updates": bool(permanent_updates and "no updates" not in permanent_updates.lower()),
            "triples_extracted": triples_extracted,
            "status": "generated",
        }

    async def extract_from_conversation(self, text: str, is_user: bool = True) -> list[str]:
        """Real-time fact extraction from a conversation message.

        Two layers, both operating on structure-stripped content:
        1. Fast regex (English-capitalization hint): every message.
        2. LLM triple extraction (multilingual): messages with 50+ words.

        Returns list of [[wiki-links]] found or created.
        """
        if not text or len(text) < 10:
            return []

        from agntspace.text import extract_entity_candidates

        links: list[str] = []

        # Layer 1: regex candidates. All markdown structure is stripped
        # inside extract_entity_candidates; candidates cannot span newlines.
        for name in extract_entity_candidates(text):
            links.append(f"[[{name}]]")
            if self._graph:
                # Registry sanitizer is the final chokepoint and will reject
                # any residual junk (e.g., control chars) without callers
                # needing to know about it.
                self._graph.add_entity(
                    name,
                    entity_type="entity",
                    authority="inferred" if is_user else "system",
                    decay_condition="when:unaccessed:90d",
                )

        if links and self._graph:
            self._graph.save()

        # Layer 2: LLM triple extraction for substantial messages.
        # extract_triples_llm strips markdown structure internally before
        # sending to the model, so scaffolding can never confuse it.
        word_count = len(text.split())
        if word_count >= 50 and self._graph and self._llm:
            try:
                authority = "explicit" if is_user else "system"
                from agntspace.infra.timezone import local_today
                today = local_today()
                extracted = await self._graph.extract_triples_llm(
                    text,
                    source_file=f"conversation/{today}",
                    source_date=today,
                    authority=authority,
                )
                if extracted:
                    log.info("Real-time extraction: %d triples from conversation", len(extracted))
            except Exception:
                log.debug("Real-time triple extraction failed", exc_info=True)

        return links

    def get_memory_path(self, date: str) -> str:
        """Get the vault path for a daily memory file.

        Args:
            date: ISO date string in YYYY-MM-DD format.

        Raises:
            ValueError: If date is not in YYYY-MM-DD format.
        """
        parts = date.split("-")
        if len(parts) != 3:
            raise ValueError(f"Invalid date format (expected YYYY-MM-DD): {date!r}")
        year, month, _day = parts
        return f"memory/{year}/{month}/memory_{date}.md"

    def read_daily_memory(self, date: str) -> str | None:
        """Read a daily memory file."""
        path = self.get_memory_path(date)
        if self._reader.exists(path):
            return self._reader.read(path).content
        return None

    def list_memory_dates(self, limit: int = 30) -> list[str]:
        """List dates that have memory files."""
        files = self._reader.list_files("memory", "**/*.md")
        dates = []
        for f in files:
            if f.name.startswith("memory_") and f.name != "MEMORY.md":
                date = f.stem.replace("memory_", "")
                dates.append(date)
        dates.sort(reverse=True)
        return dates[:limit]

    async def _gather_activity(self, date: str) -> str:
        """Gather all activity for a given date into a text summary.

        Primary sources (user-authored):
        - User journal (thoughts, decisions, learnings)
        - User notes created/modified today
        - User reflections

        Secondary sources (agent + system):
        - Agent observations
        - Conversations (with content summaries)
        - Email status
        - Tasks, goals, trust, costs
        """
        parts = []

        # ── Primary: User-authored content ──

        # User journal
        if self._journal:
            entry = self._journal.get_entry(date)
            if entry:
                parts.append(f"## User Journal\n{entry.content}")

        # User reflections (from journal Reflections section)
        if self._journal:
            reflections = self._journal.get_section("Reflections", date)
            if reflections:
                parts.append(f"## User Reflections\n{reflections}")

            # User focus (from journal Focus section)
            focus = self._journal.get_section("Focus", date)
            if focus:
                parts.append(f"## User Focus\n{focus}")

        # ── Secondary: Agent + system ──

        # Agent activity (rich structured summary from ActivityLogger, or plain observations fallback)
        if hasattr(self, "_activity") and self._activity:
            activity_summary = self._activity.get_daily_summary(date)
            if activity_summary:
                parts.append(f"## System Activity\n{activity_summary}")
        elif self._journal:
            agent_obs = self._journal.get_agent_observations(date)
            if agent_obs:
                parts.append(f"## Agent Observations\n{agent_obs.content}")

        # Conversations (with content summaries, not just counts)
        if self._memory:
            convs = await self._memory.list_conversations(limit=50)
            day_convs = [c for c in convs if c.get("updated_at", "").startswith(date)]
            if day_convs:
                parts.append(f"## Conversations ({len(day_convs)} today)")
                for c in day_convs[:8]:
                    title = c.get("title", "Untitled")
                    msgs = await self._memory.get_messages(c["id"], limit=10)
                    if msgs:
                        # First user message = topic, last assistant message = outcome
                        first_user = next((m for m in msgs if m.role == "user"), None)
                        last_assistant = next((m for m in reversed(msgs) if m.role == "assistant"), None)
                        topic = first_user.content[:120] if first_user else "?"
                        outcome = last_assistant.content[:120] if last_assistant else "?"
                        parts.append(f"### {title}\n- Topic: {topic}\n- Outcome: {outcome}\n- Messages: {len(msgs)}")

        # Email
        if self._email and self._email.is_connected:
            try:
                count = self._email.get_inbox_count()
                parts.append(f"## Email\nInbox: {count.get('total', '?')} total, {count.get('unread', '?')} unread")
            except Exception:
                pass

        # Tasks (with more detail)
        if self._tasks:
            tasks = self._tasks.list_tasks()
            active = [t for t in tasks if t.get("status") not in ("done", "cancelled")]
            done_today = [t for t in tasks if t.get("status") == "done"]
            if tasks:
                parts.append(f"## Tasks\n{len(active)} active, {len(done_today)} completed")
                for t in active[:5]:
                    brief = t.get("brief", "")[:80]
                    parts.append(f"- [{t.get('status')}] {t.get('title', '?')}: {brief}")
                if done_today:
                    parts.append("Completed today:")
                    for t in done_today[:5]:
                        parts.append(f"- {t.get('title', '?')}")

        # Goals
        if self._coach:
            goals = self._coach.list_goals(status_filter="active")
            if goals:
                parts.append(f"## Goals\n{len(goals)} active goals")
                for g in goals:
                    parts.append(f"- {g.get('title', '?')} (due: {g.get('target_date', 'no date')})")

        # Trust — current levels + recent changes
        if self._trust:
            trust_data = self._trust.get_trust_data()
            if trust_data:
                active = [t for t in trust_data if t.get("completed", 0) > 0]
                if active:
                    parts.append("## Trust")
                    for t in active:
                        parts.append(f"- {t['domain']}: {t['level']} ({t.get('completed', 0)} tasks)")
            try:
                recent_changes = self._trust.get_recent_changes(since_date=date)
                if recent_changes:
                    parts.append("## Trust Changes Today")
                    parts.extend(recent_changes)
            except Exception:
                pass

        # Costs
        if self._costs:
            try:
                daily = await self._costs.get_daily_cost()
                if daily > 0:
                    parts.append(f"## Costs\n${daily:.2f} spent today on LLM calls")
            except Exception:
                pass

        return "\n\n".join(parts) if parts else ""

    def _write_pending_memory(self, updates: str, date: str) -> None:
        """Write proposed permanent facts to a pending file for user approval.

        Facts don't go into MEMORY.md until approved. Same pattern as reflections.

        Stamps the enclosing ``correlation_id`` in frontmatter so the
        approval endpoint can feed the user's accept/reject back into the
        evolution layer's feedback signal (matching run → posterior update).
        Silently empty when no scope is active.
        """
        from agntspace.activity.decisions import current_correlation_id
        corr_id = current_correlation_id() or ""

        pending_path = f"memory/pending/memory-updates_{date}.md"
        corr_line = f"correlation_id: \"{corr_id}\"\n" if corr_id else ""
        content = (
            f"---\ntitle: \"Pending Memory Updates — {date}\"\n"
            f"date: \"{date}\"\ntype: pending_memory\nauthor: agent\nagent: main\nstatus: pending\n"
            f"source: memory_{date}\n{corr_line}---\n\n"
            f"The following facts were extracted from today's activity.\n"
            f"Approve to add them to permanent memory (MEMORY.md).\n\n"
            f"{updates.strip()}\n"
        )
        self._writer.write(pending_path, content)
        log.info("Pending memory updates written for %s", date)

    def approve_memory_updates(self, date: str) -> bool:
        """Approve pending memory updates and add them to MEMORY.md."""
        pending_path = f"memory/pending/memory-updates_{date}.md"
        if not self._reader.exists(pending_path):
            return False

        doc = self._reader.read(pending_path)

        # Extract the facts (skip the instruction text)
        lines = doc.content.split("\n")
        facts = [line for line in lines if line.strip().startswith("- ") or line.strip().startswith("Add:") or line.strip().startswith("Update:")]
        if not facts:
            # Take everything after the instruction paragraph
            fact_text = doc.content.split("\n\n", 1)[-1].strip()
        else:
            fact_text = "\n".join(facts)

        # Add to MEMORY.md
        memory_path = "memory/MEMORY.md"
        if not self._reader.exists(memory_path):
            self._writer.write(memory_path, _MEMORY_TEMPLATE, trust_reason="memory_approve_create")
        self._writer.append(memory_path, f"\n{fact_text}\n", trust_reason="memory_approve")

        # Mark as approved
        approved_content = doc.raw.replace("status: pending", "status: approved")
        self._writer.write(pending_path, approved_content, trust_reason="memory_approve")

        # Upgrade authority: user-approved facts become explicit
        if self._graph:
            source_memory = f"memory/pending/memory-updates_{date}.md"
            upgraded = self._graph.upgrade_authority(source_memory, "explicit")
            if upgraded:
                log.info("Upgraded %d triples to explicit authority", upgraded)

        log.info("Memory updates approved for %s", date)
        return True

    def list_pending_memory(self, status: str = "pending") -> list[dict]:
        """List memory update files by status (default: pending).

        Scoped to the ``memory-updates_*.md`` naming convention so sibling
        pending files owned by other subsystems (e.g. ``relationship-evolution.md``,
        which has its own approval surface under /api/relationship/pending)
        don't leak into the memory review queue.
        """
        try:
            files = self._reader.list_files("memory/pending", "memory-updates_*.md")
        except Exception:
            return []
        results = []
        for f in files:
            try:
                doc = self._reader.read(f"memory/pending/{f.name}")
                if doc.metadata.get("status") != status:
                    continue
                # Date is authoritative from filename (memory-updates_YYYY-MM-DD.md);
                # fall back to the `source:` frontmatter only if the filename parse fails.
                stem = f.stem  # "memory-updates_2026-04-14"
                date = stem.removeprefix("memory-updates_") if stem.startswith("memory-updates_") else ""
                if not date:
                    date = doc.metadata.get("source", "").replace("memory_", "")
                results.append({
                    "path": f"memory/pending/{f.name}",
                    "date": date,
                    "title": doc.metadata.get("title", f.stem),
                })
            except Exception:
                continue
        return results

    # ── Layered Memory Compaction ──
    #
    # Each layer compacts ONLY from the layer below it:
    #   dailies → week.md → month.md → quarter.md → year.md → longterm.md
    #
    # MEMORY.md is assembled from all summary files — never LLM-rewritten as a whole.

    _SUMMARY_DIR = "memory/summaries"

    def _read_summary(self, layer: str) -> str:
        """Read a summary layer file. Returns empty string if missing."""
        path = f"{self._SUMMARY_DIR}/{layer}.md"
        if self._reader.exists(path):
            return self._reader.read(path).content
        return ""

    def _write_summary(self, layer: str, content: str, title: str) -> None:
        """Write a summary layer file with frontmatter.

        Writes TWO files:
        1. Current summary at summaries/{layer}.md — used by MEMORY.md assembly
        2. Dated archive at summaries/{layer}/{period_label}.md — permanent record

        Archive naming by layer:
        - week:     summaries/week/2026-W15.md  (~52/year)
        - month:    summaries/month/2026-04.md  (12/year)
        - quarter:  summaries/quarter/2026-Q2.md (4/year)
        - year:     summaries/year/2026.md       (1/year)
        - longterm: no archive (single evolving file)
        """
        now = datetime.now(UTC)
        model = self._get_model_name()
        date_str = now.strftime("%Y-%m-%d")
        period_label = self._period_label(layer, now)

        layer_descriptions = {
            "week": "Rolling 7-day memory summary — vivid detail of recent events, decisions, and interactions. Compacted from daily memory files.",
            "month": "Rolling 30-day memory summary — key events and patterns from the past month. Compacted from weekly summaries.",
            "quarter": "Rolling 90-day memory summary — fading detail, patterns and themes only. Compacted from monthly summaries.",
            "year": "Rolling 365-day memory summary — what defined the year. Compacted from quarterly summaries.",
            "longterm": "Permanent long-term memory — stable identity, relationships, and preferences confirmed over 3+ months. Never expires.",
        }
        description = layer_descriptions.get(layer, f"Memory summary layer: {layer}.")

        full = (
            f"---\n"
            f'title: "{title}"\n'
            f'date: "{date_str}"\n'
            f'generated_at: "{now.strftime("%H:%M UTC")}"\n'
            f"type: memory_summary\n"
            f"layer: {layer}\n"
        )
        if period_label:
            full += f"period: {period_label}\n"
        full += (
            f'updated: "{now.strftime("%Y-%m-%dT%H:%M:%SZ")}"\n'
            f"author: agent\n"
            f"agent: main\n"
            f'model: "{model}"\n'
            f'description: "{description}"\n'
            f"---\n\n"
            f"{content}"
        )

        # 1. Current summary (overwritten each cycle — MEMORY.md reads this)
        self._writer.write(f"{self._SUMMARY_DIR}/{layer}.md", full)

        # 2. Dated archive (permanent record — longterm has no archive)
        if period_label:
            archive_path = f"{self._SUMMARY_DIR}/{layer}/{period_label}.md"
            self._writer.write(archive_path, full)
            log.info("Memory archive saved: %s", archive_path)

    @staticmethod
    def _period_label(layer: str, now: datetime) -> str:
        """Generate the period label for archive filenames."""
        if layer == "week":
            week_num = now.isocalendar()[1]
            return f"{now.year}-W{week_num:02d}"
        if layer == "month":
            return now.strftime("%Y-%m")
        if layer == "quarter":
            q = (now.month - 1) // 3 + 1
            return f"{now.year}-Q{q}"
        if layer == "year":
            return str(now.year)
        return ""  # longterm — no archive

    def _gather_dailies(self, days: int) -> list[tuple[str, str]]:
        """Gather daily memory files for the last N days. Returns (date, content) pairs."""
        now = datetime.now(UTC)
        results = []
        for days_ago in range(days):
            d = (now - timedelta(days=days_ago)).strftime("%Y-%m-%d")
            content = self.read_daily_memory(d)
            if content:
                results.append((d, content))
        return results

    def _gather_approved_facts(self) -> str:
        """Gather user-confirmed facts as provenance signal."""
        parts = []
        try:
            files = self._reader.list_files("memory/pending", "*.md")
        except Exception:
            return ""
        for f in files:
            try:
                doc = self._reader.read(f"memory/pending/{f.name}")
                if doc.metadata.get("status") == "approved":
                    date = doc.metadata.get("source", "").replace("memory_", "")
                    parts.append(f"### User-confirmed facts ({date})\n{doc.content[:300]}")
            except Exception:
                continue
        return "\n\n".join(parts) if parts else ""

    async def compact_memory(self, layer: str | None = None) -> dict:
        """Run layered memory compaction.

        If layer is specified, compact only that layer.
        If layer is None, compact all layers that are due:
        - week: always (runs on daily cycle)
        - month: if week.md has changed since last month compact
        - quarter: if month.md has changed since last quarter compact
        - year: if quarter.md has changed since last year compact
        - longterm: if year.md has changed since last longterm compact

        Also evaluates graph decay and reweight.

        Wrapped in ``scoped_operation`` (Rule #17 Tier B) so every layer
        write + graph reweight produced inside shares one
        ``memory-compact-*`` correlation id.
        """
        from agntspace.activity.scoped import scoped_operation

        _target = f"memory/summaries/{layer or 'all'}.md"
        async with scoped_operation(
            writer=self._writer,
            contract=self._contract,
            decisions=self._decisions,
            activity=self._activity,
            actor="main",
            action_type="memory_compact",
            action_target=_target,
            reason="memory_compact",
            scope_prefix="memory-compact",
            rationale=f"Memory compaction ({layer or 'all layers'})",
            caller="daily_cycle",
            skill_context=["memory-consolidate"],
        ) as _scope:
            return await self._do_compact_memory(layer, _scope)

    async def _do_compact_memory(self, layer: str | None, _scope: object) -> dict:
        """Internal compaction body — caller owns the scoped_operation."""
        # Evaluate decay + reweight before compaction
        decay_stats = {}
        if self._graph:
            decay_stats = self._graph.evaluate_decay()
            self._graph.reweight()

        results: dict[str, str] = {}

        if layer:
            # Compact a single specific layer
            result = await self._compact_layer(layer)
            results[layer] = result
        else:
            # Compact all layers bottom-up: week first, then month, etc.
            # Each layer reads from the layer below, so order matters.
            for lyr in ("week", "month", "quarter", "year", "longterm"):
                try:
                    result = await self._compact_layer(lyr)
                    results[lyr] = result
                except Exception as exc:
                    log.warning("Compaction of %s layer failed: %s", lyr, exc)
                    results[lyr] = "error"
                    # Log activity event so the failure is visible in /decisions
                    if self._activity and hasattr(self._activity, "log"):
                        try:
                            self._activity.log(
                                category="memory",
                                action="compaction_failed",
                                summary=f"Memory compaction of {lyr} layer failed: {exc}",
                                importance="medium",
                            )
                        except Exception:
                            pass

        # Assemble MEMORY.md from all summary files
        self._assemble_memory_md()

        return {"status": "compacted", "layers": results, "decay": decay_stats}

    def _load_compaction_config(self) -> dict:
        """Load compaction config from memory-consolidate skill YAML.

        Returns the full config dict with ``layers``, ``provenance_rules``,
        and ``output_constraint``. Falls back to ``_LAYER_CONFIG`` when the
        skill loader or YAML is unavailable.
        """
        if not self._skill_loader:
            return {}
        try:
            cfg = self._skill_loader.load_skill_config_file(
                "memory-consolidate", "compaction.yaml",
            )
            return cfg if isinstance(cfg, dict) else {}
        except Exception:
            return {}

    async def _compact_layer(self, layer: str) -> str:
        """Compact a single memory layer via the main agent."""
        # Try skill-driven config first, fall back to hardcoded _LAYER_CONFIG
        yaml_cfg = self._load_compaction_config()
        yaml_layers = yaml_cfg.get("layers", {}) if yaml_cfg else {}
        provenance = yaml_cfg.get("provenance_rules", "") if yaml_cfg else ""
        output_constraint = yaml_cfg.get("output_constraint", "") if yaml_cfg else ""

        if layer in yaml_layers:
            lc = yaml_layers[layer]
            config = {
                "prompt": lc.get("prompt", "").strip(),
                "title": lc.get("title", layer.title()),
                "source_desc": lc.get("source_desc", ""),
                "max_tokens": lc.get("max_tokens", 768),
            }
            # Append provenance rules + output constraint from YAML
            if provenance:
                config["prompt"] += f"\n{provenance}"
            if output_constraint:
                config["prompt"] += f"\n{output_constraint}"
        else:
            config = _LAYER_CONFIG.get(layer)

        if not config:
            return "unknown_layer"

        # Gather source content from the layer below
        source_content = self._gather_layer_source(layer, config)
        if not source_content.strip():
            log.info("Skipping %s compaction: no source material available", layer)
            return "no_source"

        # Read current summary for this layer (context for the agent)
        current = self._read_summary(layer)

        # Include approved facts for ALL layers (provenance must survive compaction)
        approved = self._gather_approved_facts()
        if approved:
            approved = f"\n\n## User-Confirmed Facts (highest epistemic authority — preserve [user-confirmed] tags)\n{approved}"

        context = (
            f"Current {layer} summary:\n{current or '(empty — first compaction)'}\n\n"
            f"Source material from {config['source_desc']}:\n{source_content}"
            f"{approved}"
        )

        task_instruction = (
            f"Compact the {layer} memory layer. {config['prompt']}\n\n"
            "Preserve [[wiki-links]], [user-confirmed] tags, and date markers. "
            "Output ONLY markdown bullet points. Never output JSON or tool calls.\n\n"
            "CRITICAL: NEVER fabricate, invent, or hallucinate information. "
            "Every person, project, task, and fact you mention MUST appear in "
            "the source material below. If the source is thin, produce a "
            "shorter summary — do NOT fill gaps with invented content."
        )

        # Track activity so constellation shows memory compaction
        if self._orchestrator and hasattr(self._orchestrator, "_active_tasks"):
            self._orchestrator._active_tasks["main"] = f"Memory compaction ({layer})"
        try:
            response_text = await self._agent.process_task(task=task_instruction, context=context, model_tier="capable")
        finally:
            if self._orchestrator and hasattr(self._orchestrator, "_active_tasks"):
                self._orchestrator._active_tasks.pop("main", None)

        # Sanitize — local models sometimes output raw JSON instead of markdown
        from agntspace.automation.daily_cycle import sanitize_llm_output
        response_text = sanitize_llm_output(response_text)

        # ── Fabrication detection (Rule #16) ──────────────────────
        # Memory compaction is especially vulnerable: a fabrication at
        # one layer becomes "source data" for the next. Check strictly.
        try:
            from agntspace.automation.fabrication_detector import check_memory_fabrication
            fab_report = check_memory_fabrication(response_text, source_content, layer=layer, vault_reader=self._reader)
            if fab_report.was_modified:
                response_text = fab_report.clean_output
                log.warning(
                    "Memory compaction '%s': stripped %d fabricated names: %s",
                    layer, len(fab_report.fabricated_names),
                    ", ".join(fab_report.fabricated_names),
                )
                activity = getattr(self, "_activity", None)
                if activity:
                    try:
                        activity.log(
                            category="system",
                            action="fabrication_stripped",
                            summary=f"Fabricated names removed from memory-{layer}: {', '.join(fab_report.fabricated_names)}",
                            details={
                                "layer": layer,
                                "fabricated_names": fab_report.fabricated_names,
                                "flagged_lines": fab_report.flagged_lines[:10],
                            },
                            importance="high",
                        )
                    except Exception:
                        pass
        except Exception as fab_exc:
            log.debug("Fabrication check failed for memory-%s: %s", layer, fab_exc)

        self._write_summary(layer, response_text, config["title"])
        log.info("Memory layer '%s' compacted (%d chars)", layer, len(response_text))
        return "compacted"

    def _gather_layer_source(self, layer: str, config: dict) -> str:
        """Gather source content for a specific layer."""
        parts = []

        if layer == "week":
            # Source: last 7 days of daily memory files
            # Use generous slices — truncation at 600 chars was losing critical facts
            for date, content in self._gather_dailies(7):
                parts.append(f"### {date}\n{content[:2000]}")

        elif layer == "month":
            # Source: current week.md + last 4 weeks of dailies (8-30 days ago)
            week = self._read_summary("week")
            if week:
                parts.append(f"## Current Week Summary\n{week}")
            for date, content in self._gather_dailies(30):
                # Only include days older than 7 (week already covers recent)
                now = datetime.now(UTC)
                d = datetime.fromisoformat(date)
                if (now.date() - d.date()).days > 7:
                    parts.append(f"### {date}\n{content[:1200]}")

        elif layer == "quarter":
            # Source: current month.md
            month = self._read_summary("month")
            if month:
                parts.append(f"## Current Month Summary\n{month}")
            # Also check for previous quarter summary for context
            prev = self._read_summary("quarter")
            if prev:
                parts.append(f"## Previous Quarter Summary (for continuity)\n{prev}")

        elif layer == "year":
            # Source: current quarter.md
            quarter = self._read_summary("quarter")
            if quarter:
                parts.append(f"## Current Quarter Summary\n{quarter}")
            prev = self._read_summary("year")
            if prev:
                parts.append(f"## Previous Year Summary (for continuity)\n{prev}")

        elif layer == "longterm":
            # Source: current year.md + existing longterm (to preserve identity facts)
            year = self._read_summary("year")
            if year:
                parts.append(f"## Current Year Summary\n{year}")
            prev = self._read_summary("longterm")
            if prev:
                parts.append(f"## Existing Long-term Memory (preserve unless outdated)\n{prev}")

        return "\n\n".join(parts)

    def _assemble_memory_md(self) -> None:
        """Assemble MEMORY.md from all summary layer files.

        MEMORY.md is a VIEW — concatenation of summaries — not a source of truth.
        This is what the agent reads on boot for context.

        Crash-safe: writes to a temp file first, then overwrites. If assembly fails
        partway through, the previous MEMORY.md remains intact.
        """
        from agntspace.infra.timezone import local_today
        now = local_today()
        sections = []

        for layer, heading in (
            ("week", "This Week"),
            ("month", "This Month"),
            ("quarter", "This Quarter"),
            ("year", "This Year"),
            ("longterm", "Long-term"),
        ):
            content = self._read_summary(layer)
            if content:
                sections.append(f"## {heading}\n\n{content}")
            else:
                sections.append(f"## {heading}\n\n*(no data yet)*")

        assembled = "\n\n".join(sections)
        full_content = (
            f"---\ntitle: \"Memory\"\ndate: \"{now[:10]}\"\nupdated: \"{now}\"\nauthor: agent\nagent: main\n"
            f"note: \"Assembled view — do not edit directly. Source: memory/summaries/\"\n"
            f"---\n\n{assembled}\n"
        )

        # Write to temp file first, then overwrite — crash-safe
        try:
            self._writer.write("memory/.MEMORY.md.tmp", full_content)
            self._writer.write("memory/MEMORY.md", full_content)
            log.info("MEMORY.md assembled from %d layers", len(sections))
        except Exception:
            log.error("MEMORY.md assembly failed — previous version preserved")
            raise

    def read_summary(self, layer: str) -> dict:
        """Read a specific summary layer. Returns dict with content and metadata."""
        path = f"{self._SUMMARY_DIR}/{layer}.md"
        if self._reader.exists(path):
            doc = self._reader.read(path)
            return {
                "layer": layer,
                "content": doc.content,
                "updated": doc.metadata.get("updated", ""),
                "exists": True,
                "size": len(doc.content.encode("utf-8")) if doc.content else 0,
            }
        return {"layer": layer, "content": None, "updated": None, "exists": False, "size": 0}

    def list_summaries(self) -> list[dict]:
        """List all summary layer files with metadata."""
        results = []
        for layer in ("week", "month", "quarter", "year", "longterm"):
            results.append(self.read_summary(layer))
        return results

    def list_archives(self, layer: str) -> list[dict]:
        """List all archive files for a given layer, newest first.

        Returns list of dicts with period, path, date, and size.
        """
        archive_dir = f"{self._SUMMARY_DIR}/{layer}"
        try:
            files = self._reader.list_files(archive_dir, "*.md")
        except Exception:
            return []
        results = []
        for f in sorted(files, reverse=True):
            try:
                doc = self._reader.read(f"{archive_dir}/{f.name}")
                results.append({
                    "period": f.stem,  # e.g. "2026-W15", "2026-04", "2026-Q2"
                    "path": f"{archive_dir}/{f.name}",
                    "date": doc.metadata.get("date", ""),
                    "size": len(doc.content.encode("utf-8")) if doc.content else 0,
                })
            except Exception:
                continue
        return results

    def read_archive(self, layer: str, period: str) -> dict:
        """Read a specific archive file by layer and period label.

        E.g. read_archive("week", "2026-W15") → content of summaries/week/2026-W15.md
        """
        path = f"{self._SUMMARY_DIR}/{layer}/{period}.md"
        if self._reader.exists(path):
            doc = self._reader.read(path)
            return {
                "layer": layer,
                "period": period,
                "content": doc.content,
                "date": doc.metadata.get("date", ""),
                "model": doc.metadata.get("model", ""),
                "exists": True,
            }
        return {"layer": layer, "period": period, "content": None, "exists": False}

    def search_archives(self, query: str, layers: list[str] | None = None, max_results: int = 10) -> list[dict]:
        """Search across all archive files for a query string.

        Searches newest archives first. Returns matching excerpts with context.
        Used by the deep_memory_recall tool for historical queries.
        """
        if layers is None:
            layers = ["week", "month", "quarter", "year"]  # not longterm (single file)

        query_lower = query.lower()
        matches: list[dict] = []

        for layer in layers:
            archives = self.list_archives(layer)
            for archive in archives:
                if len(matches) >= max_results:
                    break
                try:
                    doc = self._reader.read(archive["path"])
                    content = doc.content
                    if not content or query_lower not in content.lower():
                        continue
                    # Extract excerpt around match
                    idx = content.lower().index(query_lower)
                    start = max(0, idx - 300)
                    end = min(len(content), idx + 300)
                    excerpt = content[start:end].strip()
                    matches.append({
                        "layer": layer,
                        "period": archive["period"],
                        "date": archive.get("date", ""),
                        "excerpt": f"...{excerpt}...",
                        "path": archive["path"],
                    })
                except Exception:
                    continue

        return matches

    @staticmethod
    def _extract_block(text: str, label: str) -> str | None:
        """Extract content from a fenced code block."""
        marker = f"```{label}"
        start = text.find(marker)
        if start == -1:
            return None
        start = text.index("\n", start) + 1
        end = text.find("```", start)
        if end == -1:
            return text[start:].strip()
        return text[start:end].strip()

    @staticmethod
    def _strip_fenced_blocks(text: str) -> str:
        """Remove all fenced code blocks from text, keeping only prose.

        When the LLM fails to produce proper labeled blocks, the raw response
        contains ```memory, ```permanent, ```triples sections. This strips them
        so only the narrative memory content remains.
        """
        # Remove fenced blocks (```label ... ```) but keep text outside them
        cleaned = re.sub(r"```\w*\n.*?```", "", text, flags=re.DOTALL)
        # Also remove standalone ``` markers and block labels
        cleaned = re.sub(r"```\w*", "", cleaned)
        # Collapse excessive blank lines
        cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
        result = cleaned.strip()
        # If stripping removed everything, return original minus obvious non-memory blocks
        if not result:
            return text.strip()
        return result


_MEMORY_TEMPLATE = """---
title: "Memory"
author: agent
agent: main
note: "Assembled view — do not edit directly. Source: memory/summaries/"
---

## This Week

*(no data yet)*

## This Month

*(no data yet)*

## This Quarter

*(no data yet)*

## This Year

*(no data yet)*

## Long-term

*(no data yet)*

"""

# ── Per-layer compaction prompts (FALLBACK — primary source is compaction.yaml) ──
# These are used only when the memory-consolidate skill config is unavailable.
# The key epistemic principle: each layer only compresses from the layer below,
# never from raw sources multiple layers down.

_PROVENANCE_RULES = """
Provenance (CRITICAL — every fact must be traceable):
- Every bullet point MUST include a date marker: (2026-04-07) or (week of 2026-03-31)
- Include the original source type in brackets where known: [journal], [conversation], [email], [observation], [user-confirmed]
- [user-confirmed] tags MUST be preserved through ALL compaction layers — NEVER drop them even when compressing. A fact that was user-confirmed at the daily level must still carry [user-confirmed] at the yearly and longterm levels.
- When compressing multiple events into a pattern, cite the date range: (2026-01 to 2026-03)
- Never create a bullet point that cannot be traced back to at least a date and source type
- Use [[wiki-links]] for people, companies, projects, concepts
- Example: "- [[Sarah Chen]] joined [[Acme Corp]] as CTO (2026-03-15) [conversation, user-confirmed]"
"""

_WEEK_PROMPT = f"""You are the AgntSpace weekly memory compactor. Summarize the last 7 days into a vivid weekly summary.

## What to Include
- Key events, conversations, decisions WITH specific dates and context
- People encountered, relationships formed or changed
- Tasks completed or started, goals progressed
- Important information learned or shared
- Problems encountered and how they were resolved

Style: vivid, full detail. This is the freshest layer — preserve specifics.
Max 15 bullet points. Drop trivial events (routine emails, no-content interactions).
{_PROVENANCE_RULES}
Output ONLY markdown bullet points (- item). Never output JSON, tool calls, or code fences."""

_MONTH_PROMPT = f"""You are the AgntSpace monthly memory compactor. Compress the weekly summary and older dailies into a clear monthly overview.

What to preserve:
- Key decisions and their outcomes
- Relationship changes (new contacts, deepened connections)
- Project milestones and status changes
- Patterns (recurring meetings, workflows, preferences discovered)
- Important facts learned about people, companies, or topics

What to drop:
- Day-to-day operational detail (specific email counts, routine tasks)
- Events that led nowhere or were resolved completely
- Duplicate information already captured at higher layers

Style: clear but compressed. Patterns > individual events.
Max 12 bullet points.
{_PROVENANCE_RULES}
Output ONLY markdown bullet points (- item). Never output JSON, tool calls, or code fences."""

_QUARTER_PROMPT = f"""You are the AgntSpace quarterly memory compactor. Distill the monthly summary into patterns and milestones.

What to preserve:
- Major milestones and turning points
- Relationship patterns (who matters, who faded)
- Project arcs (started, pivoted, completed, abandoned)
- Decisions that shaped the quarter
- Preferences and working patterns that emerged

What to drop:
- Individual events (unless they were turning points)
- Resolved problems
- Operational detail

Style: fading — patterns and milestones only. Think "what defined this quarter?"
Max 10 bullet points.
{_PROVENANCE_RULES}
Output ONLY markdown bullet points (- item). Never output JSON, tool calls, or code fences."""

_YEAR_PROMPT = f"""You are the AgntSpace yearly memory compactor. Distill the quarterly summary into what defined this year.

What to preserve:
- Major life/work transitions
- Key relationships (new, deepened, or ended)
- Projects that shipped or were abandoned
- Decisions with lasting impact
- Skills learned, habits formed
- Goals achieved or redirected

What to drop:
- Anything that didn't have lasting impact
- Quarter-specific operational patterns

Style: reflective — "what defined this year?" Not a timeline, but a character sketch of the period.
Max 8 bullet points.
{_PROVENANCE_RULES}
Output ONLY markdown bullet points (- item). Never output JSON, tool calls, or code fences."""

_LONGTERM_PROMPT = f"""You are the AgntSpace long-term memory compactor. Maintain the permanent record of who this person is.

This is IDENTITY memory — facts that define the user regardless of time:
- Core relationships (family, close colleagues, mentors)
- Career identity (role, company, expertise)
- Preferences and values (work style, tool choices, principles)
- Life facts (location, background, languages)
- Long-running goals and aspirations

Rules:
- Only ADD facts that have been stable for 3+ months or are marked [user-confirmed]
- Only REMOVE facts that have been explicitly superseded (e.g., changed jobs)
- When in doubt, keep — this layer is conservative
- Merge the existing long-term memory with new yearly facts
- This is the most epistemically conservative layer

Style: permanent, factual, identity-defining. Think "what would a new agent need to know?"
Max 15 bullet points.
{_PROVENANCE_RULES}
Output ONLY markdown bullet points (- item). Never output JSON, tool calls, or code fences."""

_LAYER_CONFIG: dict[str, dict] = {
    "week": {
        "prompt": _WEEK_PROMPT,
        "title": "Weekly Summary",
        "source_desc": "daily memory files (last 7 days)",
        "max_tokens": 1024,
    },
    "month": {
        "prompt": _MONTH_PROMPT,
        "title": "Monthly Summary",
        "source_desc": "weekly summary + older dailies (8-30 days)",
        "max_tokens": 1024,
    },
    "quarter": {
        "prompt": _QUARTER_PROMPT,
        "title": "Quarterly Summary",
        "source_desc": "monthly summary",
        "max_tokens": 768,
    },
    "year": {
        "prompt": _YEAR_PROMPT,
        "title": "Yearly Summary",
        "source_desc": "quarterly summary",
        "max_tokens": 768,
    },
    "longterm": {
        "prompt": _LONGTERM_PROMPT,
        "title": "Long-term Memory",
        "source_desc": "yearly summary + existing long-term memory",
        "max_tokens": 768,
    },
}
