"""Markdown structure stripping — syntactic, language-agnostic.

The rule this module enforces:

    Markdown scaffolding (section headers, list bullets, code fences,
    blockquotes, YAML frontmatter, HTML tags) is never semantic content.
    Strip it before any downstream parser (entity extractor, LLM prompt,
    search indexer) sees the text.

All rules match markdown *syntax*, not vocabulary. A Swedish
``## Användarjournal``, a Japanese ``## 日記``, and an English
``## User Journal`` are all handled by the same ``^#{1,6}`` rule — the
language of the content is irrelevant.
"""

from __future__ import annotations

import re

# ── Structural match patterns ────────────────────────────────────────────────

# YAML frontmatter at the top of a document: `---\n...\n---\n`.
_YAML_FRONTMATTER = re.compile(r"\A---\n.*?\n---\n", re.DOTALL)

# Fenced code blocks (backtick or tilde fences, multi-line).
_FENCED_CODE_BLOCK = re.compile(
    r"^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n.*?^[ \t]{0,3}\1[ \t]*$",
    re.DOTALL | re.MULTILINE,
)

# HTML tags and comments. The tag regex is intentionally permissive —
# the goal is to remove anything that looks tag-shaped, not to parse HTML.
_HTML_TAG = re.compile(r"<[^>]+>")
_HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)

# Per-line structural markers:
#   - ATX headers: # ... ######
#   - Unordered list bullets: - * +
#   - Ordered list markers: 1. 2) 3.
#   - Blockquote markers: >
#   - Setext underlines / horizontal rules: --- ===
#   - Table rows: | cell | cell |
_STRUCTURE_LINE = re.compile(
    r"(?m)"
    r"(^[ \t]{0,3}#{1,6}[^\n]*$"          # ATX headers
    r"|^[ \t]{0,3}[-*+][ \t]+"             # unordered list bullet
    r"|^[ \t]{0,3}\d+[.)][ \t]+"           # ordered list marker
    r"|^[ \t]{0,3}>[ \t]?"                 # blockquote marker
    r"|^[ \t]*[-=]{3,}[ \t]*$"             # setext underline / horizontal rule
    r"|^[ \t]*\|.*\|[ \t]*$)"              # table row
)


def strip_markdown_structure(text: str) -> str:
    """Remove markdown scaffolding so downstream parsers see prose only.

    Idempotent. Language-agnostic (operates on syntax, not vocabulary).
    Every stripped span is replaced with a space so adjacent words remain
    separate tokens; this prevents a stripped header from fusing with the
    next line's first word (the bug that produced ``"Focus\\nNot"`` as a
    single entity).

    What this removes:
    - YAML frontmatter at the top of the document
    - Fenced code blocks (content + fences)
    - HTML tags and HTML comments
    - ATX headers (``# ... ######``)
    - List bullets (ordered and unordered) at line start
    - Blockquote markers at line start
    - Setext underlines / horizontal rules
    - Table rows (crude heuristic)

    What this preserves:
    - Prose words, punctuation, emphasis characters around words
    - Line structure between blocks (replaced with whitespace)
    - Non-ASCII characters (any language)
    """
    if not text:
        return ""
    cleaned = _YAML_FRONTMATTER.sub(" ", text)
    cleaned = _FENCED_CODE_BLOCK.sub(" ", cleaned)
    cleaned = _HTML_COMMENT.sub(" ", cleaned)
    cleaned = _HTML_TAG.sub(" ", cleaned)
    cleaned = _STRUCTURE_LINE.sub(" ", cleaned)
    return cleaned
