from __future__ import annotations

import asyncio
import json
import math
import hashlib
import os
import re
import unicodedata
from collections import Counter
from typing import List, Optional, Dict, Any
from urllib.parse import urlparse

import openai
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel

router = APIRouter(prefix="/dataset")


# ── License lookup ─────────────────────────────────────────────────────────────

LICENSE_NORM: dict[str, dict] = {
    # ── Creative Commons ──────────────────────────────────────────────────────────
    "cc0":                   {"spdx": "CC0-1.0",            "family": "CC",       "use": "open",       "commercial": True,      "derivatives": True},
    "cc-by":                 {"spdx": "CC-BY-4.0",          "family": "CC",       "use": "open",       "commercial": True,      "derivatives": True},
    "cc-by-2.0":             {"spdx": "CC-BY-2.0",          "family": "CC",       "use": "open",       "commercial": True,      "derivatives": True},
    "cc-by-3.0":             {"spdx": "CC-BY-3.0",          "family": "CC",       "use": "open",       "commercial": True,      "derivatives": True},
    "cc-by-4.0":             {"spdx": "CC-BY-4.0",          "family": "CC",       "use": "open",       "commercial": True,      "derivatives": True},
    "cc-by-sa":              {"spdx": "CC-BY-SA-4.0",       "family": "CC",       "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cc-by-sa-2.0":          {"spdx": "CC-BY-SA-2.0",       "family": "CC",       "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cc-by-sa-3.0":          {"spdx": "CC-BY-SA-3.0",       "family": "CC",       "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cc-by-sa-4.0":          {"spdx": "CC-BY-SA-4.0",       "family": "CC",       "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cc-by-nc":              {"spdx": "CC-BY-NC-4.0",       "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": True},
    "cc-by-nc-2.0":          {"spdx": "CC-BY-NC-2.0",       "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": True},
    "cc-by-nc-3.0":          {"spdx": "CC-BY-NC-3.0",       "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": True},
    "cc-by-nc-4.0":          {"spdx": "CC-BY-NC-4.0",       "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": True},
    "cc-by-nd":              {"spdx": "CC-BY-ND-4.0",       "family": "CC",       "use": "restricted", "commercial": True,      "derivatives": False},
    "cc-by-nd-4.0":          {"spdx": "CC-BY-ND-4.0",       "family": "CC",       "use": "restricted", "commercial": True,      "derivatives": False},
    "cc-by-nc-sa":           {"spdx": "CC-BY-NC-SA-4.0",    "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": "share-alike"},
    "cc-by-nc-sa-3.0":       {"spdx": "CC-BY-NC-SA-3.0",    "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": "share-alike"},
    "cc-by-nc-sa-4.0":       {"spdx": "CC-BY-NC-SA-4.0",    "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": "share-alike"},
    "cc-by-nc-nd":           {"spdx": "CC-BY-NC-ND-4.0",    "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": False},
    "cc-by-nc-nd-4.0":       {"spdx": "CC-BY-NC-ND-4.0",    "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": False},
    # ── Permissive OSI ────────────────────────────────────────────────────────────
    "mit":                   {"spdx": "MIT",                 "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "mit-0":                 {"spdx": "MIT-0",               "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "apache-2.0":            {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "apache-1.1":            {"spdx": "Apache-1.1",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "bsd-2-clause":          {"spdx": "BSD-2-Clause",        "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "bsd-3-clause":          {"spdx": "BSD-3-Clause",        "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "bsd-4-clause":          {"spdx": "BSD-4-Clause",        "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "isc":                   {"spdx": "ISC",                 "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "artistic-2.0":          {"spdx": "Artistic-2.0",        "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "zlib":                  {"spdx": "Zlib",                "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "boost-1.0":             {"spdx": "BSL-1.0",             "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "eupl-1.1":              {"spdx": "EUPL-1.1",            "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "eupl-1.2":              {"spdx": "EUPL-1.2",            "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "mpl-2.0":               {"spdx": "MPL-2.0",             "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cddl-1.0":              {"spdx": "CDDL-1.0",            "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "epl-1.0":               {"spdx": "EPL-1.0",             "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "epl-2.0":               {"spdx": "EPL-2.0",             "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "eupl-2.0":              {"spdx": "EUPL-2.0",            "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    # ── Copyleft ─────────────────────────────────────────────────────────────────
    "gpl-2.0":               {"spdx": "GPL-2.0-only",        "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "gpl-2.0-only":          {"spdx": "GPL-2.0-only",        "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "gpl-2.0-or-later":      {"spdx": "GPL-2.0-or-later",    "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "gpl-3.0":               {"spdx": "GPL-3.0-only",        "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "gpl-3.0-only":          {"spdx": "GPL-3.0-only",        "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "gpl-3.0-or-later":      {"spdx": "GPL-3.0-or-later",    "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "lgpl-2.0":              {"spdx": "LGPL-2.0-only",       "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "lgpl-2.1":              {"spdx": "LGPL-2.1-only",       "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "lgpl-3.0":              {"spdx": "LGPL-3.0-only",       "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "agpl-3.0":              {"spdx": "AGPL-3.0-only",       "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    "agpl-3.0-only":         {"spdx": "AGPL-3.0-only",       "family": "OSI",      "use": "copyleft",   "commercial": True,      "derivatives": "share-alike"},
    # ── Public Domain ─────────────────────────────────────────────────────────────
    "unlicense":             {"spdx": "Unlicense",           "family": "PD",       "use": "open",       "commercial": True,      "derivatives": True},
    "wtfpl":                 {"spdx": "WTFPL",               "family": "PD",       "use": "open",       "commercial": True,      "derivatives": True},
    "public-domain":         {"spdx": "LicenseRef-public-domain", "family": "PD",  "use": "open",       "commercial": True,      "derivatives": True},
    "pddl":                  {"spdx": "PDDL-1.0",            "family": "PD",       "use": "open",       "commercial": True,      "derivatives": True},
    "odbl":                  {"spdx": "ODbL-1.0",            "family": "PD",       "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    # ── Data-specific ─────────────────────────────────────────────────────────────
    "odc-by":                {"spdx": "ODC-By-1.0",          "family": "ODC",      "use": "open",       "commercial": True,      "derivatives": True},
    "odc-odbl":              {"spdx": "ODbL-1.0",            "family": "ODC",      "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    "cdla-permissive-1.0":   {"spdx": "CDLA-Permissive-1.0", "family": "CDLA",     "use": "open",       "commercial": True,      "derivatives": True},
    "cdla-permissive-2.0":   {"spdx": "CDLA-Permissive-2.0", "family": "CDLA",     "use": "open",       "commercial": True,      "derivatives": True},
    "cdla-sharing-1.0":      {"spdx": "CDLA-Sharing-1.0",    "family": "CDLA",     "use": "open",       "commercial": True,      "derivatives": "share-alike"},
    # ── OpenRAIL / AI-safety licenses ─────────────────────────────────────────────
    "openrail":              {"spdx": "OpenRAIL",            "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "openrail++":            {"spdx": "OpenRAIL++",          "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "openrail-m":            {"spdx": "OpenRAIL-M",          "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "openrail-d":            {"spdx": "OpenRAIL-D",          "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "openrail-c":            {"spdx": "OpenRAIL-C",          "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "bigscience-openrail-m": {"spdx": "BigScience-OpenRAIL-M","family": "OpenRAIL","use": "restricted", "commercial": True,      "derivatives": True},
    "bigcode-openrail-m":    {"spdx": "BigCode-OpenRAIL-M",  "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "creativeml-openrail-m": {"spdx": "CreativeML-OpenRAIL-M","family": "OpenRAIL","use": "restricted", "commercial": True,      "derivatives": True},
    "bloom-rail-1.0":        {"spdx": "BLOOM-RAIL-1.0",      "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    # ── AI Model custom licenses ───────────────────────────────────────────────────
    "llama2":                {"spdx": "LLaMA-2",             "family": "Custom",   "use": "restricted", "commercial": "limited", "derivatives": "limited"},
    "llama3":                {"spdx": "LLaMA-3",             "family": "Custom",   "use": "restricted", "commercial": "limited", "derivatives": "limited"},
    "llama4":                {"spdx": "LLaMA-4",             "family": "Custom",   "use": "restricted", "commercial": "limited", "derivatives": "limited"},
    "gemma":                 {"spdx": "Gemma",               "family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "gemma2":                {"spdx": "Gemma-2",             "family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "gemma3":                {"spdx": "Gemma-3",             "family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "gemma4":                {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "mistral":               {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "mistral-research":      {"spdx": "MRL",                 "family": "Custom",   "use": "restricted", "commercial": False,     "derivatives": "limited"},
    "qwen":                  {"spdx": "Tongyi-Qianwen-2.0",  "family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "qwen3":                 {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "deepseek":              {"spdx": "DeepSeek-License-2.0","family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "deepseek-r1":           {"spdx": "MIT",                 "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "falcon":                {"spdx": "TII-Falcon-1.0",      "family": "Custom",   "use": "restricted", "commercial": True,      "derivatives": True},
    "falcon3":               {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "yi":                    {"spdx": "Yi-License",          "family": "Custom",   "use": "restricted", "commercial": False,     "derivatives": "limited"},
    "yi-1.5":                {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "cohere-command":        {"spdx": "CC-BY-NC-4.0",        "family": "CC",       "use": "restricted", "commercial": False,     "derivatives": True},
    "grok":                  {"spdx": "Apache-2.0",          "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "stable-diffusion":      {"spdx": "CreativeML-OpenRAIL-M","family": "OpenRAIL","use": "restricted", "commercial": True,      "derivatives": True},
    "starcoder":             {"spdx": "BigCode-OpenRAIL-M",  "family": "OpenRAIL", "use": "restricted", "commercial": True,      "derivatives": True},
    "phi-3":                 {"spdx": "MIT",                 "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    "phi-4":                 {"spdx": "MIT",                 "family": "OSI",      "use": "open",       "commercial": True,      "derivatives": True},
    # ── Other ─────────────────────────────────────────────────────────────────────
    "other":                 {"spdx": "Other",               "family": "Other",    "use": "unknown",    "commercial": None,      "derivatives": None},
    "proprietary":           {"spdx": "Proprietary",         "family": "Other",    "use": "restricted", "commercial": False,     "derivatives": False},
    "unknown":               {"spdx": "Unknown",             "family": "Other",    "use": "unknown",    "commercial": None,      "derivatives": None},
}

LICENSE_ALIASES: dict[str, str] = {
    # Creative Commons
    "creative commons zero": "cc0",
    "creative commons public domain": "cc0",
    "public domain": "public-domain",
    "public-domain dedication": "cc0",
    "cc 0": "cc0",
    "cc zero": "cc0",
    "cc by": "cc-by",
    "cc by 4.0": "cc-by-4.0",
    "cc by 3.0": "cc-by-3.0",
    "cc by 2.0": "cc-by-2.0",
    "cc by-sa": "cc-by-sa",
    "cc by sa": "cc-by-sa",
    "cc by-sa 4.0": "cc-by-sa-4.0",
    "cc by-sa 3.0": "cc-by-sa-3.0",
    "cc by-nc": "cc-by-nc",
    "cc by nc": "cc-by-nc",
    "cc by-nc 4.0": "cc-by-nc-4.0",
    "cc by-nd": "cc-by-nd",
    "cc by nd": "cc-by-nd",
    "cc by-nc-sa": "cc-by-nc-sa",
    "cc by nc sa": "cc-by-nc-sa",
    "cc by-nc-sa 4.0": "cc-by-nc-sa-4.0",
    "cc by-nc-nd": "cc-by-nc-nd",
    "cc by nc nd": "cc-by-nc-nd",
    # Apache
    "apache": "apache-2.0",
    "apache2": "apache-2.0",
    "apache 2": "apache-2.0",
    "apache license 2.0": "apache-2.0",
    "apache software license": "apache-2.0",
    # GPL
    "gpl": "gpl-3.0",
    "gpl2": "gpl-2.0",
    "gpl3": "gpl-3.0",
    "gnu gpl": "gpl-3.0",
    "gnu gpl v3": "gpl-3.0",
    "gnu gpl v2": "gpl-2.0",
    "gnu general public license": "gpl-3.0",
    "lgpl": "lgpl-2.1",
    "lgpl2": "lgpl-2.0",
    "lgpl3": "lgpl-3.0",
    "gnu lgpl": "lgpl-2.1",
    "agpl": "agpl-3.0",
    "gnu agpl": "agpl-3.0",
    # BSD
    "bsd": "bsd-3-clause",
    "bsd2": "bsd-2-clause",
    "bsd3": "bsd-3-clause",
    "bsd 2": "bsd-2-clause",
    "bsd 3": "bsd-3-clause",
    "new bsd": "bsd-3-clause",
    "simplified bsd": "bsd-2-clause",
    # MIT
    "mit license": "mit",
    "the mit license": "mit",
    # OpenRAIL
    "openrail-m": "openrail-m",
    "bigscience-openrail": "bigscience-openrail-m",
    "creativeml openrail": "creativeml-openrail-m",
    "creativeml-openrail": "creativeml-openrail-m",
    # LLaMA
    "llama 2": "llama2",
    "llama 3": "llama3",
    "llama 4": "llama4",
    "llama-2": "llama2",
    "llama-3": "llama3",
    "llama-4": "llama4",
    "meta llama 2": "llama2",
    "meta llama 3": "llama3",
    # Gemma
    "google gemma": "gemma",
    "gemma terms": "gemma",
    "gemma terms of use": "gemma",
    # Other AI
    "mistral research license": "mistral-research",
    "mrl": "mistral-research",
    "tongyi qianwen": "qwen",
    "deepseek license": "deepseek",
    "tii falcon": "falcon",
    "bloom rail": "bloom-rail-1.0",
    # Data licenses
    "open database license": "odc-odbl",
    "odbl": "odc-odbl",
    "community data license agreement": "cdla-permissive-2.0",
}

def _classify_license(raw: str | None) -> dict:
    if not raw:
        return {"raw": None, "spdx": None, "family": None, "use": "unknown", "commercial": None, "derivatives": None, "confident": False}
    key = raw.lower().strip().replace("_", "-")
    if key in LICENSE_NORM:
        return {"raw": raw, **LICENSE_NORM[key], "confident": True}
    if key in LICENSE_ALIASES:
        return {"raw": raw, **LICENSE_NORM[LICENSE_ALIASES[key]], "confident": True}
    best = max(
        ((k, v) for k, v in LICENSE_NORM.items() if k in key),
        key=lambda x: len(x[0]),
        default=None,
    )
    if best:
        return {"raw": raw, **best[1], "confident": False}
    return {"raw": raw, "spdx": raw, "family": "Other", "use": "unknown", "commercial": None, "derivatives": None, "confident": False}


# ── Jurisdiction heuristics ────────────────────────────────────────────────────


# GDPR adequacy: EU member = full GDPR. Adequacy decisions (as of 2026):
# Full: Andorra, Argentina, Faroe Islands, Guernsey, Israel, Isle of Man, Japan,
#       Jersey, New Zealand, Switzerland, UK, Uruguay, Brazil (partial).
# Partial: Canada (commercial only), US (DPF enrollees only), Korea (conditional).
# EEA non-EU (GDPR applies): Norway, Iceland, Liechtenstein.

TLD_JURISDICTION: dict[str, dict] = {
    # ── EU member states (full GDPR) ─────────────────────────────────────────────
    ".eu":  {"country": None, "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".de":  {"country": "DE", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".fr":  {"country": "FR", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".nl":  {"country": "NL", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".es":  {"country": "ES", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".it":  {"country": "IT", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".pl":  {"country": "PL", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".se":  {"country": "SE", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".fi":  {"country": "FI", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".pt":  {"country": "PT", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".be":  {"country": "BE", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".at":  {"country": "AT", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".cz":  {"country": "CZ", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".ro":  {"country": "RO", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".hu":  {"country": "HU", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".dk":  {"country": "DK", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".ie":  {"country": "IE", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".gr":  {"country": "GR", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".bg":  {"country": "BG", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".hr":  {"country": "HR", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".sk":  {"country": "SK", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".si":  {"country": "SI", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".lt":  {"country": "LT", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".lv":  {"country": "LV", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".ee":  {"country": "EE", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".lu":  {"country": "LU", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".cy":  {"country": "CY", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    ".mt":  {"country": "MT", "region": "EU",  "gdpr": True,  "adequacy": "eu_member"},
    # ── EEA non-EU (GDPR applies) ─────────────────────────────────────────────────
    ".no":  {"country": "NO", "region": "EEA", "gdpr": True,  "adequacy": "eea"},
    ".is":  {"country": "IS", "region": "EEA", "gdpr": True,  "adequacy": "eea"},
    ".li":  {"country": "LI", "region": "EEA", "gdpr": True,  "adequacy": "eea"},
    # ── EU adequacy decisions (transfers allowed) ─────────────────────────────────
    ".uk":    {"country": "GB", "region": "GB",   "gdpr": False, "adequacy": "adequate"},
    ".co.uk": {"country": "GB", "region": "GB",   "gdpr": False, "adequacy": "adequate"},
    ".ch":  {"country": "CH", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    ".jp":  {"country": "JP", "region": "APAC", "gdpr": False, "adequacy": "adequate"},
    ".nz":  {"country": "NZ", "region": "APAC", "gdpr": False, "adequacy": "adequate"},
    ".il":  {"country": "IL", "region": "ME",  "gdpr": False, "adequacy": "adequate"},
    ".ad":  {"country": "AD", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    ".ar":  {"country": "AR", "region": "LATAM","gdpr": False, "adequacy": "adequate"},
    ".uy":  {"country": "UY", "region": "LATAM","gdpr": False, "adequacy": "adequate"},
    ".gg":  {"country": "GG", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    ".je":  {"country": "JE", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    ".im":  {"country": "IM", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    ".fo":  {"country": "FO", "region": "EU",  "gdpr": False, "adequacy": "adequate"},
    # ── Partial adequacy ──────────────────────────────────────────────────────────
    ".ca":  {"country": "CA", "region": "CA",   "gdpr": False, "adequacy": "partial"},
    ".us":  {"country": "US", "region": "US",   "gdpr": False, "adequacy": "partial"},
    ".kr":  {"country": "KR", "region": "APAC", "gdpr": False, "adequacy": "partial"},
    # ── Americas ──────────────────────────────────────────────────────────────────
    ".gov": {"country": "US", "region": "US",   "gdpr": False, "adequacy": None},
    ".edu": {"country": "US", "region": "US",   "gdpr": False, "adequacy": None},
    ".mil": {"country": "US", "region": "US",   "gdpr": False, "adequacy": None},
    ".br":  {"country": "BR", "region": "LATAM","gdpr": False, "adequacy": None},
    ".mx":  {"country": "MX", "region": "LATAM","gdpr": False, "adequacy": None},
    ".co":  {"country": "CO", "region": "LATAM","gdpr": False, "adequacy": None},
    ".cl":  {"country": "CL", "region": "LATAM","gdpr": False, "adequacy": None},
    ".pe":  {"country": "PE", "region": "LATAM","gdpr": False, "adequacy": None},
    ".ve":  {"country": "VE", "region": "LATAM","gdpr": False, "adequacy": None},
    ".ec":  {"country": "EC", "region": "LATAM","gdpr": False, "adequacy": None},
    ".bo":  {"country": "BO", "region": "LATAM","gdpr": False, "adequacy": None},
    ".py":  {"country": "PY", "region": "LATAM","gdpr": False, "adequacy": None},
    ".cu":  {"country": "CU", "region": "LATAM","gdpr": False, "adequacy": None},
    ".gt":  {"country": "GT", "region": "LATAM","gdpr": False, "adequacy": None},
    ".pa":  {"country": "PA", "region": "LATAM","gdpr": False, "adequacy": None},
    ".cr":  {"country": "CR", "region": "LATAM","gdpr": False, "adequacy": None},
    ".do":  {"country": "DO", "region": "LATAM","gdpr": False, "adequacy": None},
    ".hn":  {"country": "HN", "region": "LATAM","gdpr": False, "adequacy": None},
    ".ni":  {"country": "NI", "region": "LATAM","gdpr": False, "adequacy": None},
    ".sv":  {"country": "SV", "region": "LATAM","gdpr": False, "adequacy": None},
    # ── Asia-Pacific ──────────────────────────────────────────────────────────────
    ".cn":  {"country": "CN", "region": "APAC", "gdpr": False, "adequacy": None},
    ".in":  {"country": "IN", "region": "APAC", "gdpr": False, "adequacy": None},
    ".au":  {"country": "AU", "region": "APAC", "gdpr": False, "adequacy": None},
    ".sg":  {"country": "SG", "region": "APAC", "gdpr": False, "adequacy": None},
    ".hk":  {"country": "HK", "region": "APAC", "gdpr": False, "adequacy": None},
    ".tw":  {"country": "TW", "region": "APAC", "gdpr": False, "adequacy": None},
    ".id":  {"country": "ID", "region": "APAC", "gdpr": False, "adequacy": None},
    ".my":  {"country": "MY", "region": "APAC", "gdpr": False, "adequacy": None},
    ".th":  {"country": "TH", "region": "APAC", "gdpr": False, "adequacy": None},
    ".vn":  {"country": "VN", "region": "APAC", "gdpr": False, "adequacy": None},
    ".ph":  {"country": "PH", "region": "APAC", "gdpr": False, "adequacy": None},
    ".pk":  {"country": "PK", "region": "APAC", "gdpr": False, "adequacy": None},
    ".bd":  {"country": "BD", "region": "APAC", "gdpr": False, "adequacy": None},
    ".lk":  {"country": "LK", "region": "APAC", "gdpr": False, "adequacy": None},
    ".np":  {"country": "NP", "region": "APAC", "gdpr": False, "adequacy": None},
    ".mm":  {"country": "MM", "region": "APAC", "gdpr": False, "adequacy": None},
    ".kh":  {"country": "KH", "region": "APAC", "gdpr": False, "adequacy": None},
    ".mn":  {"country": "MN", "region": "APAC", "gdpr": False, "adequacy": None},
    ".kz":  {"country": "KZ", "region": "APAC", "gdpr": False, "adequacy": None},
    ".uz":  {"country": "UZ", "region": "APAC", "gdpr": False, "adequacy": None},
    ".az":  {"country": "AZ", "region": "APAC", "gdpr": False, "adequacy": None},
    ".ge":  {"country": "GE", "region": "APAC", "gdpr": False, "adequacy": None},
    # ── Middle East ───────────────────────────────────────────────────────────────
    ".sa":  {"country": "SA", "region": "ME",   "gdpr": False, "adequacy": None},
    ".ae":  {"country": "AE", "region": "ME",   "gdpr": False, "adequacy": None},
    ".eg":  {"country": "EG", "region": "ME",   "gdpr": False, "adequacy": None},
    ".tr":  {"country": "TR", "region": "ME",   "gdpr": False, "adequacy": None},
    ".ir":  {"country": "IR", "region": "ME",   "gdpr": False, "adequacy": None},
    ".iq":  {"country": "IQ", "region": "ME",   "gdpr": False, "adequacy": None},
    ".sy":  {"country": "SY", "region": "ME",   "gdpr": False, "adequacy": None},
    ".jo":  {"country": "JO", "region": "ME",   "gdpr": False, "adequacy": None},
    ".lb":  {"country": "LB", "region": "ME",   "gdpr": False, "adequacy": None},
    ".kw":  {"country": "KW", "region": "ME",   "gdpr": False, "adequacy": None},
    ".qa":  {"country": "QA", "region": "ME",   "gdpr": False, "adequacy": None},
    ".bh":  {"country": "BH", "region": "ME",   "gdpr": False, "adequacy": None},
    ".om":  {"country": "OM", "region": "ME",   "gdpr": False, "adequacy": None},
    ".ye":  {"country": "YE", "region": "ME",   "gdpr": False, "adequacy": None},
    # ── Africa ────────────────────────────────────────────────────────────────────
    ".za":  {"country": "ZA", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ng":  {"country": "NG", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ke":  {"country": "KE", "region": "AF",   "gdpr": False, "adequacy": None},
    ".gh":  {"country": "GH", "region": "AF",   "gdpr": False, "adequacy": None},
    ".et":  {"country": "ET", "region": "AF",   "gdpr": False, "adequacy": None},
    ".tz":  {"country": "TZ", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ug":  {"country": "UG", "region": "AF",   "gdpr": False, "adequacy": None},
    ".rw":  {"country": "RW", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ma":  {"country": "MA", "region": "AF",   "gdpr": False, "adequacy": None},
    ".dz":  {"country": "DZ", "region": "AF",   "gdpr": False, "adequacy": None},
    ".tn":  {"country": "TN", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ly":  {"country": "LY", "region": "AF",   "gdpr": False, "adequacy": None},
    ".sd":  {"country": "SD", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ao":  {"country": "AO", "region": "AF",   "gdpr": False, "adequacy": None},
    ".cm":  {"country": "CM", "region": "AF",   "gdpr": False, "adequacy": None},
    ".sn":  {"country": "SN", "region": "AF",   "gdpr": False, "adequacy": None},
    ".ci":  {"country": "CI", "region": "AF",   "gdpr": False, "adequacy": None},
    ".zm":  {"country": "ZM", "region": "AF",   "gdpr": False, "adequacy": None},
    ".zw":  {"country": "ZW", "region": "AF",   "gdpr": False, "adequacy": None},
    ".mz":  {"country": "MZ", "region": "AF",   "gdpr": False, "adequacy": None},
    ".bw":  {"country": "BW", "region": "AF",   "gdpr": False, "adequacy": None},
    ".na":  {"country": "NA", "region": "AF",   "gdpr": False, "adequacy": None},
    # ── Russia / CIS ─────────────────────────────────────────────────────────────
    ".ru":  {"country": "RU", "region": "CIS",  "gdpr": False, "adequacy": None},
    ".ua":  {"country": "UA", "region": "CIS",  "gdpr": False, "adequacy": None},
    ".by":  {"country": "BY", "region": "CIS",  "gdpr": False, "adequacy": None},
    ".md":  {"country": "MD", "region": "CIS",  "gdpr": False, "adequacy": None},
    ".am":  {"country": "AM", "region": "CIS",  "gdpr": False, "adequacy": None},
    # ── Oceania ───────────────────────────────────────────────────────────────────
    ".pg":  {"country": "PG", "region": "APAC", "gdpr": False, "adequacy": None},
    ".fj":  {"country": "FJ", "region": "APAC", "gdpr": False, "adequacy": None},
}

LANG_JURISDICTION: dict[str, list[str]] = {
    "de": ["DE", "AT", "CH"],
    "fr": ["FR", "BE", "CH", "CA"],
    "nl": ["NL", "BE"],
    "es": ["ES", "MX", "AR", "CO", "CL"],
    "pt": ["BR", "PT"],
    "it": ["IT"],
    "pl": ["PL"],
    "sv": ["SE"],
    "no": ["NO"],
    "fi": ["FI"],
    "da": ["DK"],
    "cs": ["CZ"],
    "ro": ["RO"],
    "hu": ["HU"],
    "zh-cn": ["CN"],
    "zh-tw": ["TW", "HK"],
    "zh": ["CN", "TW"],
    "ja": ["JP"],
    "ko": ["KR"],
    "ru": ["RU"],
    "ar": ["SA", "EG", "AE"],
    "hi": ["IN"],
    "en": ["US", "GB", "AU", "CA"],
}

EU_COUNTRIES = {"AT","BE","BG","HR","CY","CZ","DK","EE","FI","FR","DE","GR",
                "HU","IE","IT","LV","LT","LU","MT","NL","PL","PT","RO","SK",
                "SI","ES","SE"}

def _jurisdiction_from_url(url: str) -> dict | None:
    try:
        parsed = urlparse(url if url.startswith("http") else "https://" + url)
        host = parsed.hostname or ""
        for tld, meta in sorted(TLD_JURISDICTION.items(), key=lambda x: -len(x[0])):
            if host.endswith(tld):
                return {**meta, "source": "domain", "host": host}
    except Exception:
        pass
    return None

def _jurisdiction_from_lang(lang: str) -> dict | None:
    countries = LANG_JURISDICTION.get(lang)
    if not countries:
        return None
    primary = countries[0]
    return {
        "country": primary,
        "region": "EU" if primary in EU_COUNTRIES else ("US" if primary == "US" else "OTHER"),
        "gdpr": primary in EU_COUNTRIES,
        "source": "language",
        "candidates": countries,
    }


# ── MinHash dedup ──────────────────────────────────────────────────────────────

def _shingle(text: str, k: int = 5) -> set[str]:
    text = re.sub(r"\s+", " ", text.lower().strip())
    return {text[i:i+k] for i in range(max(1, len(text) - k + 1))}

def _run_minhash(
    texts: list[tuple[int, str]],
    threshold: float = 0.85,
    num_perm: int = 128,
) -> dict:
    try:
        from datasketch import MinHash, MinHashLSH
    except ImportError:
        return {"error": "datasketch not installed. Run: pip install datasketch"}

    lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
    minhashes: dict[int, MinHash] = {}

    for idx, text in texts:
        m = MinHash(num_perm=num_perm)
        for shingle in _shingle(text):
            m.update(shingle.encode("utf8"))
        minhashes[idx] = m
        try:
            lsh.insert(str(idx), m)
        except ValueError:
            pass

    duplicate_groups: list[list[int]] = []
    visited: set[int] = set()

    for idx, m in minhashes.items():
        if idx in visited:
            continue
        neighbors = [int(k) for k in lsh.query(m) if int(k) != idx]
        if neighbors:
            group = sorted([idx] + neighbors)
            if not any(set(group) <= set(g) for g in duplicate_groups):
                duplicate_groups.append(group)
            visited.update(group)

    total = len(texts)
    dup_indices = {i for g in duplicate_groups for i in g}
    exact_count = sum(
        1 for i, ti in enumerate(texts)
        for j, tj in enumerate(texts)
        if i < j and ti[1].strip() == tj[1].strip()
    )

    return {
        "total_rows": total,
        "duplicate_groups": len(duplicate_groups),
        "duplicate_row_count": len(dup_indices),
        "exact_duplicates": exact_count,
        "near_duplicate_rate": round(len(dup_indices) / total * 100, 1) if total else 0,
        "groups": [
            {"row_indices": g, "size": len(g)}
            for g in sorted(duplicate_groups, key=lambda x: -len(x))[:50]
        ],
        "threshold": threshold,
    }


# ── Topic taxonomies ───────────────────────────────────────────────────────────

GENERAL_TOPICS = [
    "Science & Technology",
    "Politics & Law",
    "Health & Medicine",
    "Finance & Economics",
    "Education",
    "Entertainment & Culture",
    "Sports",
    "Religion & Philosophy",
    "Environment & Nature",
    "Other",
]

ML_DATASET_TOPICS = [
    "Instruction Following",
    "Factual Question Answering",
    "Reasoning & Logic",
    "Coding & Software",
    "Creative Writing",
    "Summarization",
    "Dialogue & Conversation",
    "Math & Formal Reasoning",
    "Safety & Alignment",
    "Other",
]

_CODE_RE     = re.compile(r"(```|def |import |class |SELECT |<\?php|function\s+\w+\s*\()", re.I)
_INSTRUCT_RE = re.compile(r"(### (instruction|input|output)|<\|im_start\||<\|user\||<s>\[INST\])", re.I)

def _detect_taxonomy(rows: list[dict], text_columns: list[str]) -> tuple[list[str], str]:
    """Return (label_list, taxonomy_name). Samples up to 200 rows."""
    sample = rows[:200]
    code_hits = 0
    for row in sample:
        text = " ".join(str(row.get(c, "")) for c in text_columns if c in row)
        if _CODE_RE.search(text) or _INSTRUCT_RE.search(text):
            code_hits += 1
    ratio = code_hits / max(len(sample), 1)
    if ratio >= 0.40:
        return ML_DATASET_TOPICS, "ml_dataset"
    return GENERAL_TOPICS, "general"


# ── Topic classifier (gpt-4o-mini) ────────────────────────────────────────────

_topic_client: Optional[openai.AsyncOpenAI] = None

def _get_topic_client() -> openai.AsyncOpenAI:
    global _topic_client
    if _topic_client is None:
        api_key = os.environ.get("OPENAI_API_KEY")
        if not api_key:
            raise RuntimeError("OPENAI_API_KEY not set — topic analysis unavailable.")
        _topic_client = openai.AsyncOpenAI(api_key=api_key)
    return _topic_client


async def _classify_text(text: str, labels: list[str]) -> dict[str, float]:
    client = _get_topic_client()
    system = (
        "You are a topic classifier. Given a text and a list of topic labels, "
        "return a JSON object mapping each label to a probability (float 0.0–1.0) "
        "that the text belongs to that topic. Probabilities should sum to 1.0. "
        "Return only valid JSON, no explanation."
    )
    user = f"Labels: {labels}\n\nText: {text[:600]}"
    try:
        resp = await client.chat.completions.create(
            model="gpt-4o-mini",
            temperature=0,
            response_format={"type": "json_object"},
            messages=[
                {"role": "system", "content": system},
                {"role": "user", "content": user},
            ],
        )
        data = json.loads(resp.choices[0].message.content)
        return {l: float(data.get(l, 0.0)) for l in labels}
    except Exception:
        return {l: 1.0 / len(labels) for l in labels}


def _aggregate_scores(per_row: list[dict], labels: list[str]) -> list[dict]:
    totals: Counter = Counter()
    for d in per_row:
        for label, score in d.items():
            totals[label] += score
    n = max(len(per_row), 1)
    averaged = {label: totals[label] / n for label in labels}
    total = sum(averaged.values()) or 1.0
    out = [
        {
            "topic": label,
            "score": round(averaged[label], 4),
            "pct": round(averaged[label] / total * 100, 1),
        }
        for label in labels
    ]
    out.sort(key=lambda x: x["score"], reverse=True)
    return out


async def _run_topic_analysis(
    rows: list[dict],
    text_columns: list[str],
    max_rows: int = 300,
    max_chars_per_row: int = 400,
    concurrency: int = 20,
) -> dict:
    if not text_columns:
        return {"error": "No text columns available for topic analysis"}

    try:
        _get_topic_client()
    except RuntimeError as e:
        return {"error": str(e)}

    labels, taxonomy = _detect_taxonomy(rows, text_columns)

    sample = rows[:max_rows]
    texts = []
    for row in sample:
        combined = " ".join(str(row.get(c, "")) for c in text_columns if c in row)
        combined = combined[:max_chars_per_row].strip()
        if combined:
            texts.append(combined)

    if not texts:
        return {"error": "No usable text found in sample"}

    sem = asyncio.Semaphore(concurrency)

    async def _bounded(text: str) -> dict[str, float]:
        async with sem:
            return await _classify_text(text, labels)

    try:
        per_row = await asyncio.gather(*[_bounded(t) for t in texts])
        distribution = _aggregate_scores(list(per_row), labels)
    except Exception as e:
        return {"error": str(e)}

    dominant = distribution[0]["topic"] if distribution else None

    return {
        "taxonomy":      taxonomy,
        "labels":        labels,
        "distribution":  distribution,
        "dominant_topic": dominant,
        "rows_analyzed": len(texts),
    }


# ── Request / response models ──────────────────────────────────────────────────

class TextQualityRequest(BaseModel):
    rows: list[dict]
    text_columns: list[str]
    url_columns: list[str] = []
    hf_license: Optional[str] = None
    max_rows: int = 500
    minhash_threshold: float = 0.85
    minhash_num_perm: int = 128
    run_dedup: bool = True
    run_lang: bool = True
    run_license: bool = True
    run_jurisdiction: bool = True
    run_topic: bool = True


# ── Route ──────────────────────────────────────────────────────────────────────

@router.post("/text-quality")
async def text_quality(req: TextQualityRequest):
    rows = req.rows[:req.max_rows]
    if not rows:
        raise HTTPException(status_code=400, detail="No rows provided")

    def _extract_text(row: dict) -> str:
        return " ".join(
            str(row[c]) for c in req.text_columns
            if c in row and row[c] is not None
        ).strip()

    def _extract_url(row: dict) -> str | None:
        for c in req.url_columns:
            v = row.get(c)
            if v and isinstance(v, str) and v.startswith("http"):
                return v
        return None

    result: dict = {}

    # ── 1. Language distribution ──────────────────────────────────────────────
    if req.run_lang:
        try:
            from langdetect import detect, DetectorFactory, LangDetectException
            DetectorFactory.seed = 0
        except ImportError:
            result["language"] = {"error": "langdetect not installed. Run: pip install langdetect"}
        else:
            lang_counts: Counter = Counter()
            lang_failures = 0
            for row in rows:
                text = _extract_text(row)
                if len(text) < 20:
                    lang_failures += 1
                    continue
                try:
                    lang_counts[detect(text)] += 1
                except LangDetectException:
                    lang_failures += 1

            total_detected = sum(lang_counts.values())
            result["language"] = {
                "total_detected": total_detected,
                "undetected": lang_failures,
                "distribution": [
                    {"lang": lang, "count": cnt, "pct": round(cnt / total_detected * 100, 1)}
                    for lang, cnt in lang_counts.most_common()
                ],
                "dominant_lang": lang_counts.most_common(1)[0][0] if lang_counts else None,
                "multilingual": len(lang_counts) > 1,
            }

    # ── 2. Duplicate / near-duplicate detection ───────────────────────────────
    if req.run_dedup:
        indexed_texts = [
            (i, _extract_text(row))
            for i, row in enumerate(rows)
            if len(_extract_text(row)) >= 10
        ]
        result["dedup"] = _run_minhash(indexed_texts, req.minhash_threshold, req.minhash_num_perm)

    # ── 3. License classification ─────────────────────────────────────────────
    if req.run_license:
        license_info = _classify_license(req.hf_license)

        license_col_candidates = [
            c for c in (rows[0].keys() if rows else [])
            if any(k in c.lower() for k in ("license", "licence", "rights", "copyright"))
        ]
        inline_licenses: Counter = Counter()
        for row in rows:
            for col in license_col_candidates:
                v = row.get(col)
                if v and isinstance(v, str):
                    inline_licenses[v.strip()] += 1

        result["license"] = {
            "dataset_license": license_info,
            "inline_license_columns": license_col_candidates,
            "inline_license_distribution": [
                {"raw": lic, "count": cnt, **_classify_license(lic)}
                for lic, cnt in inline_licenses.most_common(10)
            ],
        }

    # ── 4. Jurisdiction detection ─────────────────────────────────────────────
    if req.run_jurisdiction:
        domain_jurisdictions: Counter = Counter()
        domain_gdpr: Counter = Counter()
        url_count = 0

        for row in rows:
            url = _extract_url(row)
            if url:
                url_count += 1
                j = _jurisdiction_from_url(url)
                if j:
                    region = j.get("region") or "unknown"
                    domain_jurisdictions[region] += 1
                    if j.get("gdpr"):
                        domain_gdpr["EU/EEA"] += 1
                    else:
                        domain_gdpr["non-EU"] += 1

        lang_j = None
        dominant = (result.get("language") or {}).get("dominant_lang")
        if dominant:
            lang_j = _jurisdiction_from_lang(dominant)

        total_j = sum(domain_jurisdictions.values())
        result["jurisdiction"] = {
            "url_rows_analyzed": url_count,
            "region_distribution": [
                {"region": r, "count": c, "pct": round(c / total_j * 100, 1)}
                for r, c in domain_jurisdictions.most_common()
            ] if total_j else [],
            "gdpr_exposure": dict(domain_gdpr),
            "gdpr_pct": round(domain_gdpr.get("EU/EEA", 0) / url_count * 100, 1) if url_count else None,
            "lang_inferred_jurisdiction": lang_j,
            "note": "Domain-based when URL columns present; language-inferred otherwise." if not url_count else None,
        }

    # ── 5. Topic analysis ─────────────────────────────────────────────────────
    if req.run_topic:
        result["topic"] = await _run_topic_analysis(
            rows,
            req.text_columns,
            max_rows=req.max_rows,
        )

    return result