"""
AgentDrop Shield -- Layer 3: Text Extraction & Unicode Normalization.

Extracts human-readable text from all supported file formats and
normalizes it to defeat evasion techniques (zero-width chars, Bidi
overrides, homoglyphs, tag characters).

Supports: txt, md, csv, json, xml, yaml, html, pdf-metadata,
          png-metadata, jpeg-metadata, svg.
Uses only Python stdlib.
"""

from __future__ import annotations

import csv
import html.parser
import io
import json
import logging
import re
import unicodedata
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from typing import Generator, Optional

from .models import ScanConfig, ThreatLevel

logger = logging.getLogger("agentdrop.shield.text_extractor")


# ── Unicode constants ────────────────────────────────────────────────

ZERO_WIDTH_CHARS: frozenset[str] = frozenset([
    "\u200b",  # ZERO WIDTH SPACE
    "\u200c",  # ZERO WIDTH NON-JOINER
    "\u200d",  # ZERO WIDTH JOINER
    "\u2060",  # WORD JOINER
    "\ufeff",  # ZERO WIDTH NO-BREAK SPACE / BOM
    "\u00ad",  # SOFT HYPHEN
    "\u034f",  # COMBINING GRAPHEME JOINER
    "\u061c",  # ARABIC LETTER MARK
    "\u115f",  # HANGUL CHOSEONG FILLER
    "\u1160",  # HANGUL JUNGSEONG FILLER
    "\u17b4",  # KHMER VOWEL INHERENT AQ
    "\u17b5",  # KHMER VOWEL INHERENT AA
    "\u180e",  # MONGOLIAN VOWEL SEPARATOR
])

BIDI_CONTROL_CHARS: frozenset[str] = frozenset([
    "\u200e",  # LEFT-TO-RIGHT MARK
    "\u200f",  # RIGHT-TO-LEFT MARK
    "\u202a",  # LEFT-TO-RIGHT EMBEDDING
    "\u202b",  # RIGHT-TO-LEFT EMBEDDING
    "\u202c",  # POP DIRECTIONAL FORMATTING
    "\u202d",  # LEFT-TO-RIGHT OVERRIDE
    "\u202e",  # RIGHT-TO-LEFT OVERRIDE
    "\u2066",  # LEFT-TO-RIGHT ISOLATE
    "\u2067",  # RIGHT-TO-LEFT ISOLATE
    "\u2068",  # FIRST STRONG ISOLATE
    "\u2069",  # POP DIRECTIONAL ISOLATE
])

TAG_CHAR_RANGE = range(0xE0001, 0xE007F + 1)

# ── Homoglyph map ────────────────────────────────────────────────────
# Maps visually similar characters from other scripts to ASCII.
# This is intentionally focused on the most commonly abused homoglyphs
# in injection attacks rather than being exhaustive.

_HOMOGLYPH_MAP: dict[str, str] = {
    # Cyrillic -> Latin
    "\u0430": "a",  # а
    "\u0435": "e",  # е
    "\u043e": "o",  # о
    "\u0440": "p",  # р
    "\u0441": "c",  # с
    "\u0443": "y",  # у
    "\u0445": "x",  # х
    "\u0456": "i",  # і
    "\u0458": "j",  # ј
    "\u04bb": "h",  # һ
    "\u0410": "A",  # А
    "\u0412": "B",  # В
    "\u0415": "E",  # Е
    "\u041a": "K",  # К
    "\u041c": "M",  # М
    "\u041d": "H",  # Н
    "\u041e": "O",  # О
    "\u0420": "P",  # Р
    "\u0421": "C",  # С
    "\u0422": "T",  # Т
    "\u0425": "X",  # Х
    # Greek -> Latin
    "\u03b1": "a",  # α (alpha)
    "\u03bf": "o",  # ο (omicron)
    "\u03c1": "p",  # ρ (rho)
    "\u0391": "A",  # Α (Alpha)
    "\u0392": "B",  # Β (Beta)
    "\u0395": "E",  # Ε (Epsilon)
    "\u0397": "H",  # Η (Eta)
    "\u0399": "I",  # Ι (Iota)
    "\u039a": "K",  # Κ (Kappa)
    "\u039c": "M",  # Μ (Mu)
    "\u039d": "N",  # Ν (Nu)
    "\u039f": "O",  # Ο (Omicron)
    "\u03a1": "P",  # Ρ (Rho)
    "\u03a4": "T",  # Τ (Tau)
    "\u03a7": "X",  # Χ (Chi)
    # Fullwidth -> ASCII (NFKC handles most, but belt-and-suspenders)
    "\uff21": "A", "\uff22": "B", "\uff23": "C", "\uff24": "D",
    "\uff25": "E", "\uff26": "F", "\uff27": "G", "\uff28": "H",
    "\uff29": "I", "\uff2a": "J", "\uff2b": "K", "\uff2c": "L",
    "\uff2d": "M", "\uff2e": "N", "\uff2f": "O", "\uff30": "P",
    "\uff31": "Q", "\uff32": "R", "\uff33": "S", "\uff34": "T",
    "\uff35": "U", "\uff36": "V", "\uff37": "W", "\uff38": "X",
    "\uff39": "Y", "\uff3a": "Z",
    "\uff41": "a", "\uff42": "b", "\uff43": "c", "\uff44": "d",
    "\uff45": "e", "\uff46": "f", "\uff47": "g", "\uff48": "h",
    "\uff49": "i", "\uff4a": "j", "\uff4b": "k", "\uff4c": "l",
    "\uff4d": "m", "\uff4e": "n", "\uff4f": "o", "\uff50": "p",
    "\uff51": "q", "\uff52": "r", "\uff53": "s", "\uff54": "t",
    "\uff55": "u", "\uff56": "v", "\uff57": "w", "\uff58": "x",
    "\uff59": "y", "\uff5a": "z",
    # Armenian -> Latin (commonly abused homoglyphs)
    "\u0561": "a",  # Armenian Small Letter Ayb -> a
    "\u0562": "b",  # Armenian Small Letter Ben -> b (approximate)
    "\u0564": "d",  # Armenian Small Letter Da -> d
    "\u0565": "e",  # Armenian Small Letter Ech -> e
    "\u0566": "z",  # Armenian Small Letter Za -> z
    "\u056b": "i",  # Armenian Small Letter Ini -> i
    "\u056d": "x",  # Armenian Small Letter Xeh -> x (approximate)
    "\u056e": "c",  # Armenian Small Letter Ca -> c (approximate)
    "\u0570": "h",  # Armenian Small Letter Ho -> h
    "\u0572": "g",  # Armenian Small Letter Gim -> g (approximate)
    "\u0574": "m",  # Armenian Small Letter Men -> m (approximate)
    "\u0575": "y",  # Armenian Small Letter Yi -> y
    "\u0576": "n",  # Armenian Small Letter Now -> n
    "\u0578": "o",  # Armenian Small Letter Vo -> o (approximate)
    "\u057a": "p",  # Armenian Small Letter Peh -> p (approximate)
    "\u057d": "s",  # Armenian Small Letter Seh -> s
    "\u057e": "v",  # Armenian Small Letter Vew -> v
    "\u057f": "t",  # Armenian Small Letter Tiwn -> t (approximate)
    "\u0580": "r",  # Armenian Small Letter Reh -> r (approximate)
    "\u0582": "u",  # Armenian Small Letter Yiwn -> u (approximate)
    "\u0585": "o",  # Armenian Small Letter Oh -> o
    # Cherokee -> Latin (select commonly abused)
    "\u13a0": "D",  # Cherokee Letter A (looks like D)
    "\u13d2": "R",  # Cherokee Letter Sv (looks like R)
    "\u13b3": "W",  # Cherokee Letter La (looks like W)
    # Common look-alikes
    "\u2010": "-",  # HYPHEN
    "\u2011": "-",  # NON-BREAKING HYPHEN
    "\u2012": "-",  # FIGURE DASH
    "\u2013": "-",  # EN DASH
    "\u2014": "-",  # EM DASH
    "\u2018": "'",  # LEFT SINGLE QUOTATION MARK
    "\u2019": "'",  # RIGHT SINGLE QUOTATION MARK
    "\u201c": '"',  # LEFT DOUBLE QUOTATION MARK
    "\u201d": '"',  # RIGHT DOUBLE QUOTATION MARK
}

# Build translation table once
_HOMOGLYPH_TABLE = str.maketrans(_HOMOGLYPH_MAP)


# ── Anomaly tracking ────────────────────────────────────────────────

@dataclass
class TextAnomaly:
    """A suspicious Unicode anomaly found during extraction."""
    anomaly_type: str
    source: str
    count: int


@dataclass
class TextExtractionResult:
    """All extracted text plus Unicode anomalies."""
    texts: list[tuple[str, str]] = field(default_factory=list)
    """List of (source_label, normalized_text) pairs."""

    anomalies: list[TextAnomaly] = field(default_factory=list)
    threat_level: ThreatLevel = ThreatLevel.CLEAN
    warnings: list[str] = field(default_factory=list)
    details: dict = field(default_factory=dict)

    @property
    def full_text(self) -> str:
        """Concatenate all extracted text blocks."""
        return "\n".join(text for _, text in self.texts if text.strip())


# ── Simple HTML text extractor ──────────────────────────────────────

class _HTMLTextExtractor(html.parser.HTMLParser):
    """Minimal HTML-to-text using only stdlib."""

    def __init__(self) -> None:
        super().__init__()
        self._pieces: list[str] = []
        self._skip = False

    def handle_starttag(self, tag: str, attrs: list) -> None:
        if tag.lower() in ("script", "style"):
            self._skip = True

    def handle_endtag(self, tag: str) -> None:
        if tag.lower() in ("script", "style"):
            self._skip = False

    def handle_data(self, data: str) -> None:
        if not self._skip:
            self._pieces.append(data)

    def handle_comment(self, data: str) -> None:
        # Comments can contain injections -- extract them too
        self._pieces.append(f"[HTML_COMMENT: {data}]")

    def get_text(self) -> str:
        return " ".join(self._pieces)


class TextExtractor:
    """Layer 3 -- extract and normalize text from supported file formats.

    Normalization pipeline:
    1. Strip zero-width characters (and count them as anomalies)
    2. Strip Bidi control characters
    3. Strip Unicode tag characters (U+E0001..U+E007F)
    4. NFKC normalization
    5. Homoglyph normalization
    6. Whitespace collapse
    """

    def __init__(self, config: ScanConfig) -> None:
        self._cfg = config

    def extract(self, data: bytes, filename: str, detected_mime: Optional[str] = None) -> TextExtractionResult:
        """Extract and normalize all text from a file.

        Args:
            data: Raw file bytes.
            filename: Original filename.
            detected_mime: MIME type from Layer 1 (optional hint).

        Returns:
            TextExtractionResult with normalized text and anomalies.
        """
        result = TextExtractionResult()
        mime = detected_mime or self._guess_mime(filename)

        try:
            raw_texts = list(self._extract_by_format(data, filename, mime))
        except Exception as exc:
            logger.warning("Text extraction failed for %s: %s", filename, exc)
            result.warnings.append(f"Extraction error: {exc}")
            return result

        total_chars = 0
        for source, raw_text in raw_texts:
            if not raw_text:
                continue

            # Enforce text length limit
            remaining = self._cfg.max_text_length_chars - total_chars
            if remaining <= 0:
                result.warnings.append("Text length limit reached; truncated")
                break
            if len(raw_text) > remaining:
                raw_text = raw_text[:remaining]
                result.warnings.append(f"Text from {source} truncated at limit")

            # Detect anomalies before stripping
            anomalies = self._detect_anomalies(raw_text, source)
            result.anomalies.extend(anomalies)

            # Normalize
            normalized = self._normalize(raw_text)
            result.texts.append((source, normalized))
            total_chars += len(normalized)

        # Score anomalies
        if result.anomalies:
            total_anomaly_count = sum(a.count for a in result.anomalies)
            result.details["total_anomaly_chars"] = total_anomaly_count
            if total_anomaly_count > 50:
                result.threat_level = ThreatLevel.HIGH
                result.warnings.append(
                    f"Excessive Unicode anomalies ({total_anomaly_count} chars) "
                    "-- likely evasion attempt"
                )
            elif total_anomaly_count > 10:
                result.threat_level = ThreatLevel.MEDIUM
                result.warnings.append(
                    f"Suspicious Unicode characters detected ({total_anomaly_count} chars)"
                )
            elif total_anomaly_count > 0:
                result.threat_level = ThreatLevel.LOW
                result.warnings.append(
                    f"Minor Unicode anomalies ({total_anomaly_count} chars)"
                )

        result.details["sources"] = len(result.texts)
        result.details["total_chars"] = total_chars
        return result

    # ------------------------------------------------------------------
    # Normalization
    # ------------------------------------------------------------------

    def _normalize(self, text: str) -> str:
        """Full normalization pipeline."""
        # 1. Strip zero-width characters
        text = "".join(
            c for c in text
            if c not in ZERO_WIDTH_CHARS and ord(c) not in TAG_CHAR_RANGE
        )
        # 2. Strip Bidi overrides
        text = "".join(c for c in text if c not in BIDI_CONTROL_CHARS)
        # 3. NFKC normalization
        text = unicodedata.normalize("NFKC", text)
        # 3b. Strip combining diacritical marks (catches accent-based evasion)
        # NFD decomposition separates base chars from combining marks,
        # then we remove the combining marks and recompose.
        text_nfd = unicodedata.normalize("NFD", text)
        text = "".join(
            c for c in text_nfd
            if unicodedata.category(c) != "Mn"  # Mn = Mark, Nonspacing
        )
        text = unicodedata.normalize("NFC", text)
        # 4. Homoglyph normalization
        text = text.translate(_HOMOGLYPH_TABLE)
        # 5. Whitespace collapse (preserve newlines for line-based detection)
        text = re.sub(r"[^\S\n]+", " ", text)
        text = re.sub(r"\n{3,}", "\n\n", text)
        return text.strip()

    @staticmethod
    def _detect_anomalies(text: str, source: str) -> list[TextAnomaly]:
        """Count suspicious Unicode characters before normalization."""
        anomalies: list[TextAnomaly] = []

        zw = sum(1 for c in text if c in ZERO_WIDTH_CHARS)
        if zw > 0:
            anomalies.append(TextAnomaly("zero_width_chars", source, zw))

        bidi = sum(1 for c in text if c in BIDI_CONTROL_CHARS)
        if bidi > 0:
            anomalies.append(TextAnomaly("bidi_control_chars", source, bidi))

        tag = sum(1 for c in text if ord(c) in TAG_CHAR_RANGE)
        if tag > 0:
            anomalies.append(TextAnomaly("unicode_tag_chars", source, tag))

        return anomalies

    # ------------------------------------------------------------------
    # Format-specific extraction
    # ------------------------------------------------------------------

    def _extract_by_format(
        self, data: bytes, filename: str, mime: Optional[str]
    ) -> Generator[tuple[str, str], None, None]:
        """Route to the correct extractor based on MIME / extension."""

        ext = filename.lower().rsplit(".", maxsplit=1)[-1] if "." in filename else ""

        # Plain text family
        if mime and mime.startswith("text/") or ext in (
            "txt", "md", "log", "rst", "py", "js", "ts", "sh", "bat",
            "c", "cpp", "h", "rs", "go", "java", "rb", "css", "sql",
            "toml",
        ):
            yield from self._extract_text(data)

        elif ext == "csv" or mime == "text/csv":
            yield from self._extract_csv(data)

        elif ext == "json" or mime == "application/json":
            yield from self._extract_json(data)

        elif ext in ("xml",) or mime == "application/xml":
            yield from self._extract_xml(data)

        elif ext in ("yaml", "yml") or mime == "application/x-yaml":
            yield from self._extract_yaml(data)

        elif ext in ("html", "htm") or mime == "text/html":
            yield from self._extract_html(data)

        elif ext == "pdf" or mime == "application/pdf":
            yield from self._extract_pdf_metadata(data)

        elif ext == "svg" or mime == "image/svg+xml":
            yield from self._extract_svg(data)

        elif ext == "png" or mime == "image/png":
            yield from self._extract_png_metadata(data)

        elif ext in ("jpg", "jpeg") or mime == "image/jpeg":
            yield from self._extract_jpeg_metadata(data)

        else:
            # Fallback: try to decode as UTF-8
            yield from self._extract_text(data)

    def _extract_text(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Decode raw bytes as text, handling BOM and encoding detection."""
        text = self._decode_bytes(data)
        if text.strip():
            yield ("body", text)

    def _extract_csv(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract all cell values from CSV."""
        text = self._decode_bytes(data)
        try:
            reader = csv.reader(io.StringIO(text))
            all_cells: list[str] = []
            for row_idx, row in enumerate(reader):
                if row_idx > 100_000:  # Safety limit
                    break
                for cell in row:
                    cell = cell.strip()
                    if cell:
                        all_cells.append(cell)
            if all_cells:
                yield ("csv_cells", "\n".join(all_cells))
        except csv.Error as exc:
            logger.debug("CSV parse error: %s", exc)
            # Fall back to raw text
            if text.strip():
                yield ("body", text)

    def _extract_json(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract all string values from JSON with depth limiting."""
        text = self._decode_bytes(data)
        try:
            obj = json.loads(text)
        except (json.JSONDecodeError, ValueError):
            if text.strip():
                yield ("body", text)
            return

        strings: list[str] = []
        self._walk_json(obj, strings, depth=0)
        if strings:
            yield ("json_values", "\n".join(strings))

    def _walk_json(self, obj: object, out: list[str], depth: int) -> None:
        """Recursively collect string values from parsed JSON."""
        if depth > self._cfg.max_json_depth:
            return
        if isinstance(obj, str):
            if obj.strip():
                out.append(obj)
        elif isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(k, str) and k.strip():
                    out.append(k)
                self._walk_json(v, out, depth + 1)
        elif isinstance(obj, list):
            for item in obj:
                self._walk_json(item, out, depth + 1)

    def _extract_xml(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract text content and attribute values from XML."""
        try:
            root = ET.fromstring(data)
        except ET.ParseError:
            text = self._decode_bytes(data)
            if text.strip():
                yield ("body", text)
            return

        texts: list[str] = []
        self._walk_xml(root, texts, depth=0)
        if texts:
            yield ("xml_content", "\n".join(texts))

    def _walk_xml(self, elem: ET.Element, out: list[str], depth: int) -> None:
        """Recursively collect text from XML elements."""
        if depth > self._cfg.max_xml_depth:
            return
        # Element text
        if elem.text and elem.text.strip():
            out.append(elem.text.strip())
        # Attribute values
        for val in elem.attrib.values():
            if val.strip():
                out.append(val)
        # Tail text
        if elem.tail and elem.tail.strip():
            out.append(elem.tail.strip())
        for child in elem:
            self._walk_xml(child, out, depth + 1)

    def _extract_yaml(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract string content from YAML.

        Since PyYAML is not stdlib, we use a regex-based approach that
        captures quoted and unquoted string values.  This is not a full
        parser but catches the vast majority of injection payloads hiding
        in YAML values.
        """
        text = self._decode_bytes(data)
        if not text.strip():
            return

        # Extract quoted strings
        quoted = re.findall(r'''["']((?:[^"'\\]|\\.)*)["']''', text)
        # Also yield the full text for line-based scanning
        yield ("yaml_body", text)
        if quoted:
            yield ("yaml_strings", "\n".join(quoted))

    def _extract_html(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract visible text and comments from HTML."""
        text = self._decode_bytes(data)
        if not text.strip():
            return

        parser = _HTMLTextExtractor()
        try:
            parser.feed(text)
            extracted = parser.get_text()
        except Exception:
            extracted = text

        if extracted.strip():
            yield ("html_text", extracted)

        # Also yield raw source for pattern matching on tags/attributes
        yield ("html_raw", text)

    def _extract_pdf_metadata(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract metadata strings from PDF without external libraries.

        This does NOT extract page text (that requires a full PDF parser).
        It extracts the Info dictionary fields which are a common injection
        vector and can be read with simple byte scanning.
        """
        # PDF metadata fields in the Info dictionary
        metadata_keys = [
            b"/Title", b"/Author", b"/Subject", b"/Keywords",
            b"/Creator", b"/Producer",
        ]

        for key in metadata_keys:
            idx = data.find(key)
            if idx == -1:
                continue
            # Read the value after the key -- it is either a string literal
            # in parentheses or a hex string in angle brackets.
            after = data[idx + len(key):idx + len(key) + 1024]

            # Parenthesized string: /Title (Some Title)
            paren_match = re.search(rb"\(([^)]{1,512})\)", after)
            if paren_match:
                try:
                    val = paren_match.group(1).decode("utf-8", errors="replace")
                    yield (f"pdf_metadata:{key.decode()}", val)
                except Exception:
                    pass

            # Hex string: /Title <48656C6C6F>
            hex_match = re.search(rb"<([0-9a-fA-F]{2,1024})>", after)
            if hex_match:
                try:
                    raw = bytes.fromhex(hex_match.group(1).decode("ascii"))
                    val = raw.decode("utf-8", errors="replace")
                    yield (f"pdf_metadata:{key.decode()}", val)
                except Exception:
                    pass

    # ------------------------------------------------------------------
    # SVG extraction (enhanced)
    # ------------------------------------------------------------------

    def _extract_svg(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract all text content from SVG files.

        SVGs are XML-based but need special handling for:
        - <text> elements and nested <tspan>
        - CDATA sections (can contain hidden instructions)
        - <foreignObject> with embedded HTML
        - <script> tags
        - Attribute values (especially xlink:href, onclick, etc.)
        - <desc> and <title> elements
        """
        text = self._decode_bytes(data)
        if not text.strip():
            return

        # Full XML extraction catches element text and attributes
        try:
            root = ET.fromstring(data)
            texts: list[str] = []
            self._walk_xml(root, texts, depth=0)
            if texts:
                yield ("svg_content", "\n".join(texts))
        except ET.ParseError:
            pass

        # Also extract CDATA sections via regex (ET strips them)
        cdata_matches = re.findall(r"<!\[CDATA\[(.*?)\]\]>", text, re.DOTALL)
        for i, cdata in enumerate(cdata_matches):
            if cdata.strip():
                yield (f"svg_cdata_{i}", cdata)

        # Extract HTML content from foreignObject blocks
        foreign_matches = re.findall(
            r"<foreignObject[^>]*>(.*?)</foreignObject>", text, re.DOTALL | re.IGNORECASE
        )
        for i, foreign in enumerate(foreign_matches):
            if foreign.strip():
                yield (f"svg_foreignobject_{i}", foreign)

        # Raw source for pattern matching
        yield ("svg_raw", text)

    # ------------------------------------------------------------------
    # PNG metadata extraction
    # ------------------------------------------------------------------

    def _extract_png_metadata(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract text metadata from PNG files without external libraries.

        PNG stores text in three chunk types:
        - tEXt: uncompressed key-value text
        - zTXt: compressed text (we read the key, skip compressed value)
        - iTXt: international text with language tag

        Each chunk has: length (4 bytes) + type (4 bytes) + data + CRC (4 bytes).
        """
        if len(data) < 8 or data[:8] != b"\x89PNG\r\n\x1a\n":
            return

        pos = 8  # Skip PNG signature
        texts_found: list[tuple[str, str]] = []
        total_metadata_chars = 0

        while pos + 12 <= len(data):
            # Read chunk: 4 bytes length + 4 bytes type
            try:
                chunk_len = int.from_bytes(data[pos:pos + 4], "big")
            except (ValueError, IndexError):
                break

            chunk_type = data[pos + 4:pos + 8]
            chunk_data = data[pos + 8:pos + 8 + chunk_len]

            if chunk_type == b"tEXt":
                # Format: keyword\x00text
                null_idx = chunk_data.find(b"\x00")
                if null_idx >= 0:
                    key = chunk_data[:null_idx].decode("latin-1", errors="replace")
                    val = chunk_data[null_idx + 1:].decode("latin-1", errors="replace")
                    if val.strip():
                        texts_found.append((f"png_tEXt:{key}", val))
                        total_metadata_chars += len(val)

            elif chunk_type == b"zTXt":
                # Format: keyword\x00compression_method\x00compressed_text
                # We extract the keyword; decompressing requires zlib which
                # is stdlib, so let's do it.
                null_idx = chunk_data.find(b"\x00")
                if null_idx >= 0:
                    key = chunk_data[:null_idx].decode("latin-1", errors="replace")
                    # compression_method is 1 byte after null
                    compressed = chunk_data[null_idx + 2:]
                    if compressed:
                        try:
                            import zlib
                            decompressed = zlib.decompress(compressed)
                            val = decompressed.decode("latin-1", errors="replace")
                            if val.strip():
                                texts_found.append((f"png_zTXt:{key}", val))
                                total_metadata_chars += len(val)
                        except Exception:
                            pass

            elif chunk_type == b"iTXt":
                # Format: keyword\x00compression_flag\x00compression_method\x00
                #         language_tag\x00translated_keyword\x00text
                null_idx = chunk_data.find(b"\x00")
                if null_idx >= 0:
                    key = chunk_data[:null_idx].decode("utf-8", errors="replace")
                    rest = chunk_data[null_idx + 1:]
                    # Skip compression_flag (1 byte) + compression_method (1 byte)
                    if len(rest) >= 2:
                        compression_flag = rest[0]
                        rest = rest[2:]
                        # Skip language_tag\x00translated_keyword\x00
                        null1 = rest.find(b"\x00")
                        if null1 >= 0:
                            rest = rest[null1 + 1:]
                            null2 = rest.find(b"\x00")
                            if null2 >= 0:
                                text_data = rest[null2 + 1:]
                                if compression_flag == 1 and text_data:
                                    try:
                                        import zlib
                                        text_data = zlib.decompress(text_data)
                                    except Exception:
                                        pass
                                val = text_data.decode("utf-8", errors="replace")
                                if val.strip():
                                    texts_found.append((f"png_iTXt:{key}", val))
                                    total_metadata_chars += len(val)

            # Move to next chunk: 4 (length) + 4 (type) + chunk_len + 4 (CRC)
            pos += 12 + chunk_len

            # Safety: stop if chunk_len is unreasonable
            if chunk_len > len(data):
                break

        # Yield all found text
        for source, val in texts_found:
            yield (source, val)

        # Flag suspiciously long metadata
        if total_metadata_chars > 500:
            yield (
                "png_metadata_warning",
                f"[SUSPICIOUS: PNG contains {total_metadata_chars} chars of text metadata "
                "-- possible hidden instructions]",
            )

    # ------------------------------------------------------------------
    # JPEG metadata extraction
    # ------------------------------------------------------------------

    def _extract_jpeg_metadata(self, data: bytes) -> Generator[tuple[str, str], None, None]:
        """Extract text metadata from JPEG files without external libraries.

        Targets:
        - EXIF comments (COM marker, 0xFFFE)
        - EXIF APP1 data (0xFFE1) containing UserComment, ImageDescription
        - IPTC/APP13 (0xFFED) containing caption/description
        - XMP/APP1 (0xFFE1) containing XML metadata
        """
        if len(data) < 3 or data[:3] != b"\xff\xd8\xff":
            return

        total_metadata_chars = 0
        texts_found: list[tuple[str, str]] = []

        pos = 2  # Skip SOI marker

        while pos + 4 <= len(data):
            if data[pos] != 0xFF:
                break

            marker = data[pos + 1]

            # SOS (Start of Scan) or EOI -- stop parsing markers
            if marker == 0xDA or marker == 0xD9:
                break

            # Markers without length (standalone)
            if 0xD0 <= marker <= 0xD7 or marker == 0x01:
                pos += 2
                continue

            # Read segment length
            if pos + 4 > len(data):
                break
            seg_len = int.from_bytes(data[pos + 2:pos + 4], "big")
            seg_data = data[pos + 4:pos + 2 + seg_len]

            # COM marker (0xFE) -- JPEG comment
            if marker == 0xFE:
                comment = seg_data.decode("utf-8", errors="replace").strip()
                if comment:
                    texts_found.append(("jpeg_comment", comment))
                    total_metadata_chars += len(comment)

            # APP1 (0xE1) -- EXIF or XMP
            elif marker == 0xE1:
                # Check for XMP
                xmp_header = b"http://ns.adobe.com/xap/1.0/\x00"
                if seg_data[:len(xmp_header)] == xmp_header:
                    xmp_text = seg_data[len(xmp_header):].decode("utf-8", errors="replace")
                    if xmp_text.strip():
                        # Extract text content from XMP XML
                        xmp_strings = self._extract_xmp_strings(xmp_text)
                        for s in xmp_strings:
                            texts_found.append(("jpeg_xmp", s))
                            total_metadata_chars += len(s)

                # Check for EXIF
                elif seg_data[:6] in (b"Exif\x00\x00", b"Exif\x00\xff"):
                    exif_strings = self._extract_exif_strings(seg_data[6:])
                    for s in exif_strings:
                        texts_found.append(("jpeg_exif", s))
                        total_metadata_chars += len(s)

            # APP13 (0xED) -- IPTC
            elif marker == 0xED:
                iptc_strings = self._extract_iptc_strings(seg_data)
                for s in iptc_strings:
                    texts_found.append(("jpeg_iptc", s))
                    total_metadata_chars += len(s)

            pos += 2 + seg_len

            # Safety limit
            if pos > len(data):
                break

        for source, val in texts_found:
            yield (source, val)

        if total_metadata_chars > 500:
            yield (
                "jpeg_metadata_warning",
                f"[SUSPICIOUS: JPEG contains {total_metadata_chars} chars of text metadata "
                "-- possible hidden instructions]",
            )

    @staticmethod
    def _extract_xmp_strings(xmp_xml: str) -> list[str]:
        """Extract human-readable strings from XMP XML metadata."""
        strings: list[str] = []
        # Use regex to extract text between XML tags (good enough for metadata)
        # This catches dc:description, dc:title, dc:creator, xmp:Label, etc.
        tag_text = re.findall(r">([^<]{3,})<", xmp_xml)
        for t in tag_text:
            t = t.strip()
            if t and not t.startswith(("http://", "https://", "urn:", "uuid:")):
                strings.append(t)
        return strings

    @staticmethod
    def _extract_exif_strings(exif_data: bytes) -> list[str]:
        """Extract readable strings from raw EXIF data.

        This is a lightweight scan -- not a full TIFF/EXIF parser.
        Looks for ASCII strings of 4+ printable characters that could
        contain injection payloads in fields like ImageDescription,
        UserComment, Artist, Copyright, etc.
        """
        strings: list[str] = []
        # Scan for runs of printable ASCII characters
        current: list[int] = []
        for byte in exif_data:
            if 0x20 <= byte <= 0x7E:
                current.append(byte)
            else:
                if len(current) >= 10:  # Only strings of 10+ chars are interesting
                    s = bytes(current).decode("ascii")
                    # Filter out common EXIF noise
                    if not all(c in "0123456789:/ " for c in s):
                        strings.append(s)
                current = []
        if len(current) >= 10:
            s = bytes(current).decode("ascii")
            if not all(c in "0123456789:/ " for c in s):
                strings.append(s)
        return strings

    @staticmethod
    def _extract_iptc_strings(iptc_data: bytes) -> list[str]:
        """Extract text strings from IPTC/IIM data.

        IPTC records have format: 0x1C record dataset_size(2 bytes) data.
        We look for text-heavy datasets: caption (2:120), headline (2:105),
        keywords (2:25), special instructions (2:40), writer (2:122).
        """
        strings: list[str] = []
        # Text dataset numbers in record 2
        text_datasets = {25, 40, 55, 80, 85, 90, 101, 105, 115, 116, 120, 122}
        pos = 0

        while pos + 5 <= len(iptc_data):
            if iptc_data[pos] != 0x1C:
                pos += 1
                continue

            record = iptc_data[pos + 1]
            dataset = iptc_data[pos + 2]
            ds_len = int.from_bytes(iptc_data[pos + 3:pos + 5], "big")

            if ds_len > 0 and ds_len < 10000 and pos + 5 + ds_len <= len(iptc_data):
                if record == 2 and dataset in text_datasets:
                    val = iptc_data[pos + 5:pos + 5 + ds_len]
                    text = val.decode("utf-8", errors="replace").strip()
                    if text:
                        strings.append(text)

            pos += 5 + max(ds_len, 0)

        return strings

    # ------------------------------------------------------------------
    # Encoding helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _decode_bytes(data: bytes) -> str:
        """Decode bytes to string, handling BOM and common encodings."""
        # Check for BOM
        if data[:3] == b"\xef\xbb\xbf":
            return data[3:].decode("utf-8", errors="replace")
        if data[:2] in (b"\xff\xfe", b"\xfe\xff"):
            return data.decode("utf-16", errors="replace")
        if data[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"):
            return data.decode("utf-32", errors="replace")

        # Try UTF-8 first
        try:
            return data.decode("utf-8")
        except UnicodeDecodeError:
            pass

        # Fall back to latin-1 (never fails)
        return data.decode("latin-1")

    @staticmethod
    def _guess_mime(filename: str) -> Optional[str]:
        """Rough MIME guess from extension for routing purposes."""
        ext = filename.lower().rsplit(".", maxsplit=1)[-1] if "." in filename else ""
        mapping = {
            "txt": "text/plain", "md": "text/plain", "csv": "text/csv",
            "json": "application/json", "xml": "application/xml",
            "yaml": "application/x-yaml", "yml": "application/x-yaml",
            "html": "text/html", "htm": "text/html",
            "pdf": "application/pdf", "svg": "image/svg+xml",
        }
        return mapping.get(ext)
