"""Browser automation integration via Playwright + httpx fallback.

Provides headless Chromium for web research, content extraction,
and screenshot capture. Exposed as agent tools via the registry.

Also provides ``scrape_page_sync()`` — a synchronous helper used by
the agent scrape tools (scrape_url, research_scrape) that tries
agent-browser first then falls back to httpx + HTML text extraction
so scraping works even when agent-browser returns empty (common
for JS-rendered or protected pages).
"""

from __future__ import annotations

import asyncio
import logging
import shutil
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import suppress
from dataclasses import dataclass, field

from agntspace.text import extract_readable_text as _html_to_text

log = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Scraping strategy (skill-driven, Rule #12)
# ---------------------------------------------------------------------------
#
# Values are seeded with safe defaults. At server startup, main.py calls
# ``set_scraping_config()`` with the ``scraping:`` block from the
# ``browser-automation`` skill, letting users tune timeouts / concurrency /
# backend preference without editing code.
#
# Sticky health for agent-browser: Device Guard and corporate EDR tools
# sometimes block the agent-browser subprocess entirely. The FIRST failing
# call flips ``_agent_browser_healthy`` to False for the process lifetime,
# so we don't burn 30-120s on every subsequent URL probing a blocked binary.
# Restart the server to re-probe.


@dataclass
class ScrapingConfig:
    backend_preference: list[str] = field(default_factory=lambda: ["agent-browser", "httpx"])
    agent_browser_timeout: int = 8
    httpx_timeout: int = 15
    concurrency: int = 5
    min_content_length: int = 200
    retry_with_alternate_user_agent: bool = True
    # Per-request page limits for research_scrape batches. These are the
    # last-line fallbacks for when the skill YAML is missing or broken —
    # canonical values live in browser-automation/SKILL.md.
    default_pages: int = 5
    max_pages: int = 30


_CONFIG = ScrapingConfig()
_agent_browser_healthy: bool | None = None  # None=unprobed, True=ok, False=blocked


def set_scraping_config(cfg: dict | None) -> None:
    """Install the scraping strategy from the browser-automation skill.

    Called once at startup from main.py. Missing keys keep their defaults.
    Passing ``None`` or ``{}`` is a no-op so the module works even without
    a skill loader wired up (tests, CLI tools).
    """
    global _CONFIG
    if not cfg:
        return
    _CONFIG = ScrapingConfig(
        backend_preference=list(cfg.get("backend_preference", _CONFIG.backend_preference)),
        agent_browser_timeout=int(cfg.get("agent_browser_timeout", _CONFIG.agent_browser_timeout)),
        httpx_timeout=int(cfg.get("httpx_timeout", _CONFIG.httpx_timeout)),
        concurrency=int(cfg.get("concurrency", _CONFIG.concurrency)),
        min_content_length=int(cfg.get("min_content_length", _CONFIG.min_content_length)),
        retry_with_alternate_user_agent=bool(
            cfg.get("retry_with_alternate_user_agent", _CONFIG.retry_with_alternate_user_agent)
        ),
        default_pages=int(cfg.get("default_pages", _CONFIG.default_pages)),
        max_pages=int(cfg.get("max_pages", _CONFIG.max_pages)),
    )
    log.info(
        "Scraping config: backends=%s agent_browser=%ss httpx=%ss conc=%d min_len=%d pages=%d/%d",
        _CONFIG.backend_preference,
        _CONFIG.agent_browser_timeout,
        _CONFIG.httpx_timeout,
        _CONFIG.concurrency,
        _CONFIG.min_content_length,
        _CONFIG.default_pages,
        _CONFIG.max_pages,
    )


def get_scraping_config() -> ScrapingConfig:
    return _CONFIG


def agent_browser_health() -> str:
    """Return 'healthy', 'blocked', 'missing', or 'unprobed' for diagnostics."""
    if _resolve_agent_browser() is None:
        return "missing"
    if _agent_browser_healthy is None:
        return "unprobed"
    return "healthy" if _agent_browser_healthy else "blocked"


def _resolve_agent_browser() -> str | None:
    """Locate the ``agent-browser`` CLI. ``shutil.which`` honours PATHEXT on
    Windows, so npm-installed ``.cmd`` wrappers resolve to a full path that
    can be invoked directly without ``shell=True``."""
    return shutil.which("agent-browser")


def _try_agent_browser(url: str, *, min_len: int = 0) -> str | None:
    """Try scraping *url* via ``agent-browser`` CLI. Returns text or None.

    Honours the sticky health flag: once agent-browser has failed in this
    process, every subsequent call short-circuits to None so httpx can run
    immediately. No more 30s-per-URL-times-10-URLs death spirals.
    """
    global _agent_browser_healthy
    if _agent_browser_healthy is False:
        return None
    if "agent-browser" not in _CONFIG.backend_preference:
        return None

    exe = _resolve_agent_browser()
    if not exe:
        _agent_browser_healthy = False
        return None

    timeout = _CONFIG.agent_browser_timeout
    try:
        open_res = subprocess.run(
            [exe, "open", url],
            capture_output=True, timeout=timeout,
        )
        if open_res.returncode != 0:
            stderr = open_res.stderr.decode("utf-8", errors="replace").strip()
            log.warning("agent-browser open failed (rc=%d): %s", open_res.returncode, stderr[:300])
            # Device Guard / EDR block produces non-zero on every call — flip sticky.
            if _agent_browser_healthy is None:
                _agent_browser_healthy = False
                log.warning("agent-browser marked unhealthy for this session — using httpx only")
            return None

        for selector in ("article", "main", "body"):
            result = subprocess.run(
                [exe, "get", "text", selector],
                capture_output=True, timeout=timeout,
            )
            if result.returncode == 0 and result.stdout:
                text = result.stdout.decode("utf-8", errors="replace").strip()
                if text and len(text) > min_len:
                    _agent_browser_healthy = True
                    return text
            elif result.returncode != 0:
                stderr = result.stderr.decode("utf-8", errors="replace").strip()
                if stderr:
                    log.debug("agent-browser get text %s failed: %s", selector, stderr[:200])
    except subprocess.TimeoutExpired:
        log.warning("agent-browser timed out (%ss) for %s — marking unhealthy", timeout, url)
        _agent_browser_healthy = False
    except Exception as exc:
        log.warning("agent-browser error for %s: %s", url, exc)
        _agent_browser_healthy = False
    return None


_UA_POOL = [  # arch:deterministic — realistic browser User-Agent strings for HTTP-fallback scraping; technical rotation pool, not user-tunable content policy
    # Chrome on Windows
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    # Firefox on Windows
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
    # Safari on macOS
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
]


def _httpx_fetch(
    url: str, ua: str, *, min_len: int = 0, extract_images: bool = False,
) -> tuple[str | None, str | None, list[str] | None]:
    """Single httpx fetch attempt.

    Returns ``(text, error, image_urls)``. When *extract_images* is False
    the third element is always None (backward compat for callers that
    don't care about images).
    """
    try:
        import httpx
    except ImportError:
        return None, "httpx not installed", None

    from agntspace.infra.i18n import get_locale

    user_locale = (get_locale() or "en").split("_")[0]
    accept_language = "en;q=0.9" if user_locale == "en" else f"{user_locale},en;q=0.5"

    headers = {
        "User-Agent": ua,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": accept_language,
    }
    try:
        with httpx.Client(timeout=_CONFIG.httpx_timeout, follow_redirects=True, verify=False) as client:
            resp = client.get(url, headers=headers)
            if resp.status_code >= 400:
                return None, f"HTTP {resp.status_code}", None
            ct = resp.headers.get("content-type", "")
            if "html" not in ct and "text" not in ct:
                return None, f"non-HTML content-type: {ct}", None

            raw_html = resp.text

            # Extract image URLs from raw HTML before stripping tags.
            image_urls: list[str] | None = None
            if extract_images:
                try:
                    from agntspace.kb.image_resolver import extract_image_urls_from_html
                    image_urls = extract_image_urls_from_html(raw_html, base_url=url)
                except Exception:
                    image_urls = []

            text = _html_to_text(raw_html)
            if text and len(text) > min_len:
                return text, None, image_urls
            return None, f"extracted {len(text) if text else 0} chars (below min {min_len})", image_urls
    except Exception as exc:
        return None, f"{type(exc).__name__}: {exc}", None


def _try_httpx(
    url: str, *, min_len: int = 0, extract_images: bool = False,
) -> tuple[str | None, str | None, list[str] | None]:
    """Fallback: fetch page via httpx. Retries once with an alternate UA if the
    first attempt hits an anti-bot 403/429. Returns (text, error, image_urls)."""
    if "httpx" not in _CONFIG.backend_preference:
        return None, "httpx disabled by config", None

    text, err, imgs = _httpx_fetch(url, _UA_POOL[0], min_len=min_len, extract_images=extract_images)
    if text:
        return text, None, imgs

    if _CONFIG.retry_with_alternate_user_agent and err and any(
        s in err for s in ("403", "429", "blocked", "forbidden")
    ):
        log.info("httpx retrying %s with alternate UA after: %s", url, err)
        text, err2, imgs2 = _httpx_fetch(url, _UA_POOL[1], min_len=min_len, extract_images=extract_images)
        if text:
            return text, None, imgs2
        err = f"{err} → retry: {err2}"

    return None, err, None


def scrape_page_sync(url: str, *, min_len: int | None = None) -> str:
    """Scrape a URL and return its text content. Empty string on total failure.

    Backend order is dictated by the scraping config loaded from the
    browser-automation skill. Once agent-browser has been marked unhealthy
    in this process it's skipped entirely — no 30-second probes against a
    blocked binary on every URL in a batch.
    """
    effective_min = _CONFIG.min_content_length if min_len is None else min_len

    for backend in _CONFIG.backend_preference:
        if backend == "agent-browser":
            text = _try_agent_browser(url, min_len=effective_min)
            if text:
                return text
        elif backend == "httpx":
            text, err, _imgs = _try_httpx(url, min_len=effective_min)
            if text:
                return text
            log.info("httpx failed for %s: %s", url, err)
    return ""


def scrape_page_detailed(url: str, *, min_len: int | None = None) -> tuple[str, str | None]:
    """Same as ``scrape_page_sync`` but returns a ``(text, last_error)`` tuple
    so callers (research_scrape) can surface WHY a scrape failed instead of
    silently returning empty strings."""
    effective_min = _CONFIG.min_content_length if min_len is None else min_len
    last_error: str | None = None

    for backend in _CONFIG.backend_preference:
        if backend == "agent-browser":
            text = _try_agent_browser(url, min_len=effective_min)
            if text:
                return text, None
            health = agent_browser_health()
            if health != "healthy":
                last_error = f"agent-browser {health}"
        elif backend == "httpx":
            text, err, _imgs = _try_httpx(url, min_len=effective_min)
            if text:
                return text, None
            if err:
                last_error = f"httpx: {err}"
    return "", last_error


def scrape_with_images(
    url: str, *, min_len: int | None = None, max_images: int = 10,
) -> tuple[str, str | None, list[str]]:
    """Scrape a URL and also extract image URLs from the page.

    Returns ``(text, last_error, image_urls)``. Image URLs are extracted
    from the raw HTML ``<img>`` tags before the HTML is stripped to text.
    The caller is responsible for downloading the images (via
    ``kb.image_resolver.download_images``).

    Only the httpx backend supports image extraction — agent-browser
    returns text only and image_urls will be empty if agent-browser
    succeeds first.
    """
    effective_min = _CONFIG.min_content_length if min_len is None else min_len
    last_error: str | None = None
    all_image_urls: list[str] = []

    for backend in _CONFIG.backend_preference:
        if backend == "agent-browser":
            text = _try_agent_browser(url, min_len=effective_min)
            if text:
                return text, None, []
            health = agent_browser_health()
            if health != "healthy":
                last_error = f"agent-browser {health}"
        elif backend == "httpx":
            text, err, imgs = _try_httpx(
                url, min_len=effective_min, extract_images=True,
            )
            if imgs:
                all_image_urls = imgs[:max_images]
            if text:
                return text, None, all_image_urls
            if err:
                last_error = f"httpx: {err}"

    return "", last_error, all_image_urls


def scrape_many(urls: list[str], *, min_len: int | None = None) -> list[dict]:
    """Scrape multiple URLs in parallel.

    Returns a list of ``{url, text, error}`` dicts in the SAME order as the
    input. Concurrency is governed by the skill config. The thread pool is
    created per-call so a lingering batch can't leak threads into the
    event loop.
    """
    if not urls:
        return []
    effective_min = _CONFIG.min_content_length if min_len is None else min_len
    concurrency = max(1, min(_CONFIG.concurrency, len(urls)))

    results: list[dict] = [{"url": u, "text": "", "error": None} for u in urls]
    with ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix="scrape") as pool:
        future_to_idx = {
            pool.submit(scrape_page_detailed, u, min_len=effective_min): i
            for i, u in enumerate(urls)
        }
        for fut in as_completed(future_to_idx):
            idx = future_to_idx[fut]
            try:
                text, err = fut.result()
            except Exception as exc:
                text, err = "", f"{type(exc).__name__}: {exc}"
            results[idx]["text"] = text
            results[idx]["error"] = err
    return results

# Lazy-loaded browser instance
_browser = None
_playwright = None
_lock = asyncio.Lock()


async def _get_browser():
    """Get or create the shared browser instance."""
    global _browser, _playwright
    async with _lock:
        if _browser and _browser.is_connected():
            return _browser
        try:
            from playwright.async_api import async_playwright
            _playwright = await async_playwright().start()
            _browser = await _playwright.chromium.launch(headless=True)
            log.info("Browser launched (headless Chromium)")
            return _browser
        except Exception:
            log.exception("Failed to launch browser")
            return None


async def close_browser():
    """Shut down the browser instance."""
    global _browser, _playwright
    if _browser:
        with suppress(Exception):
            await _browser.close()
        _browser = None
    if _playwright:
        with suppress(Exception):
            await _playwright.stop()
        _playwright = None


def is_connected() -> bool:
    """Check if browser is available."""
    return _browser is not None and _browser.is_connected()


async def navigate_and_read(url: str, *, wait_for: str = "domcontentloaded", timeout: int = 15000) -> dict:
    """Navigate to a URL and extract the page content as text.

    Returns: {url, title, content, links[]}
    """
    browser = await _get_browser()
    if not browser:
        return {"error": "Browser not available. Playwright may not be installed."}

    page = await browser.new_page()
    try:
        response = await page.goto(url, wait_until=wait_for, timeout=timeout)
        if not response:
            return {"error": f"No response from {url}"}

        title = await page.title()

        # Extract main text content (strip nav, ads, etc.)
        content = await page.evaluate("""() => {
            // Remove noise elements
            for (const sel of ['nav', 'header', 'footer', 'aside', '.sidebar', '.nav', '.menu', '.ad', '.cookie', '[role="banner"]', '[role="navigation"]']) {
                document.querySelectorAll(sel).forEach(el => el.remove());
            }
            // Get main content or body
            const main = document.querySelector('main, article, [role="main"], .content, .post, .entry') || document.body;
            return main.innerText.substring(0, 8000);
        }""")

        # Extract links
        links = await page.evaluate("""() => {
            return Array.from(document.querySelectorAll('a[href]'))
                .map(a => ({ text: a.innerText.trim().substring(0, 80), href: a.href }))
                .filter(l => l.text && l.href.startsWith('http'))
                .slice(0, 20);
        }""")

        return {
            "url": page.url,
            "title": title,
            "content": content,
            "links": links,
            "status": response.status,
        }
    except Exception as e:
        return {"error": f"Navigation failed: {e!s}"}
    finally:
        await page.close()


async def screenshot(url: str, *, full_page: bool = False, timeout: int = 15000) -> dict:
    """Take a screenshot of a URL.

    Returns: {url, title, path} where path is a temp file.
    """
    import tempfile
    from pathlib import Path

    browser = await _get_browser()
    if not browser:
        return {"error": "Browser not available."}

    page = await browser.new_page(viewport={"width": 1280, "height": 720})
    try:
        await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
        title = await page.title()

        path = Path(tempfile.mktemp(suffix=".png"))
        await page.screenshot(path=str(path), full_page=full_page)

        return {"url": page.url, "title": title, "path": str(path)}
    except Exception as e:
        return {"error": f"Screenshot failed: {e!s}"}
    finally:
        await page.close()


async def search_web(query: str) -> dict:
    """Search the web using DuckDuckGo (no API key needed).

    Returns: {query, results: [{title, url, snippet}]}
    """
    browser = await _get_browser()
    if not browser:
        return {"error": "Browser not available."}

    page = await browser.new_page()
    try:
        search_url = f"https://duckduckgo.com/?q={query.replace(' ', '+')}&t=h_&ia=web"
        await page.goto(search_url, wait_until="domcontentloaded", timeout=15000)

        # Wait for results to load
        await page.wait_for_selector("[data-result]", timeout=5000).catch_(lambda _: None)

        results = await page.evaluate("""() => {
            const items = document.querySelectorAll('[data-result], .result, .nrn-react-div');
            return Array.from(items).slice(0, 8).map(el => {
                const a = el.querySelector('a[href]');
                const snippet = el.querySelector('.snippet, [data-result] + div, .result__snippet');
                return {
                    title: a ? a.innerText.trim() : '',
                    url: a ? a.href : '',
                    snippet: snippet ? snippet.innerText.trim().substring(0, 200) : '',
                };
            }).filter(r => r.title && r.url && !r.url.includes('duckduckgo'));
        }""")

        return {"query": query, "results": results}
    except Exception as e:
        return {"error": f"Search failed: {e!s}"}
    finally:
        await page.close()
