"""AISA guided assessment questionnaire.

Interactive questionnaire that evaluates an agentic system against
AISA layers through targeted yes/no/partial questions.
"""

from dataclasses import dataclass


@dataclass
class Question:
    """A single assessment question."""
    criterion_id: str
    text: str
    help_text: str = ""


# ---------------------------------------------------------------------------
# Questions organized by layer/contract
# ---------------------------------------------------------------------------

QUESTIONS: dict[str, list[Question]] = {
    "L1": [
        Question("L1.1",
                 "Does your system support multiple LLM providers (e.g., OpenAI, Anthropic, Google)?",
                 "Model adapter flexibility allows swapping providers without rewriting logic."),
        Question("L1.2",
                 "Do you use structured prompt templates (with variables, versioning, or reuse)?",
                 "Templates separate prompt logic from content and enable systematic testing."),
        Question("L1.3",
                 "Do you manage context window limits (truncation, summarization, or token budgeting)?",
                 "Without this, long conversations silently lose context or fail."),
        Question("L1.4",
                 "Are safety filters or grounding mechanisms in place (content moderation, citation, RAG)?",
                 "Filters prevent harmful outputs; grounding reduces hallucination."),
        Question("L1.5",
                 "Have you applied fine-tuning, alignment, or instruction tuning to your models?",
                 "Domain adaptation can improve quality and reduce prompt complexity."),
        Question("L1.6",
                 "Does your system produce structured outputs (JSON, typed objects, Pydantic models)?",
                 "Structured outputs are more reliable and easier to validate downstream."),
    ],
    "L2": [
        Question("L2.1",
                 "Are your tools/functions formally defined with schemas (names, descriptions, parameters)?",
                 "Formal schemas let the LLM understand tool capabilities and reduce errors."),
        Question("L2.2",
                 "Do you validate tool inputs and outputs against schemas?",
                 "Schema validation catches malformed calls before they cause side effects."),
        Question("L2.3",
                 "Are tool executions sandboxed or isolated (Docker, VM, restricted environment)?",
                 "Sandboxing prevents untrusted code from affecting the host system."),
        Question("L2.4",
                 "Do you enforce per-tool or per-agent permission scopes?",
                 "Least-privilege access prevents agents from performing unauthorized actions."),
        Question("L2.5",
                 "Does your system use MCP (Model Context Protocol) for tool integration?",
                 "MCP provides a standardized protocol for tool discovery and invocation."),
        Question("L2.6",
                 "Do you have error handling and retry logic for failed tool calls?",
                 "Graceful failure handling prevents cascading errors in agentic workflows."),
        Question("L2.7",
                 "Do you enforce rate limiting on external API calls?",
                 "Rate limiting prevents runaway agents from exhausting API quotas or budgets."),
    ],
    "L3": [
        Question("L3.1",
                 "Can your agent plan and decompose complex tasks into sub-tasks?",
                 "Planning enables handling of complex, multi-step objectives."),
        Question("L3.2",
                 "Does your agent maintain coherent reasoning across multiple turns?",
                 "Multi-turn coherence is essential for complex problem-solving."),
        Question("L3.3",
                 "Does your system have explicit memory management (short-term, long-term, episodic)?",
                 "Memory enables learning, context retention, and personalization."),
        Question("L3.4",
                 "Does your agent track goals and progress toward completion?",
                 "Goal tracking prevents drift, infinite loops, and scope creep."),
        Question("L3.5",
                 "Does your agent self-evaluate or reflect on its outputs before finalizing?",
                 "Reflection loops catch errors and improve output quality."),
        Question("L3.6",
                 "Can your agent incorporate external feedback (human or system) during execution?",
                 "Feedback integration enables course correction and human oversight."),
    ],
    "L4": [
        Question("L4.1",
                 "Do you use workflow orchestration (DAGs, state machines, pipelines)?",
                 "Orchestration provides predictable, auditable execution flows."),
        Question("L4.2",
                 "Is agent state persisted and recoverable (checkpointing, save/load)?",
                 "State persistence enables recovery from failures and long-running tasks."),
        Question("L4.3",
                 "Does your system coordinate multiple agents (delegation, handoff, collaboration)?",
                 "Multi-agent coordination enables specialization and parallel execution."),
        Question("L4.4",
                 "Do you have observability and tracing (distributed traces, spans)?",
                 "Tracing enables debugging, performance analysis, and audit trails."),
        Question("L4.5",
                 "Is structured logging in place with correlation IDs?",
                 "Structured logs enable search, alerting, and incident investigation."),
        Question("L4.6",
                 "Do you manage agent lifecycles (creation, scaling, teardown)?",
                 "Lifecycle management prevents resource leaks and enables scaling."),
        Question("L4.7",
                 "Do you optimize for latency and cost (caching, batching, routing)?",
                 "Optimization prevents unnecessary cost and improves user experience."),
    ],
    "L5": [
        Question("L5.1",
                 "Do you test individual components (prompts, tools, retrieval) in isolation?",
                 "Component testing catches issues before they compound in the full system."),
        Question("L5.2",
                 "Do you evaluate end-to-end agent trajectories (multi-step task completion)?",
                 "Trajectory evaluation captures emergent behaviors that unit tests miss."),
        Question("L5.3",
                 "Do you monitor for behavioral drift or anomalies in production?",
                 "Drift detection catches silent degradation from model or data changes."),
        Question("L5.4",
                 "Do you run regression tests to catch behavioral regressions?",
                 "Regression tests ensure changes don't break existing capabilities."),
        Question("L5.5",
                 "Do you have human-in-the-loop evaluation workflows?",
                 "Human eval catches subjective quality issues automation misses."),
        Question("L5.6",
                 "Do you track quality metrics (accuracy, faithfulness, relevancy)?",
                 "Metrics enable data-driven improvement and stakeholder communication."),
    ],
    "L6": [
        Question("L6.1",
                 "Are your agent configurations versioned (prompts, tools, workflows)?",
                 "Versioning enables rollback, comparison, and reproducibility."),
        Question("L6.2",
                 "Do you have CI/CD pipelines for your agent system?",
                 "CI/CD ensures changes are tested and deployed consistently."),
        Question("L6.3",
                 "Can you run A/B tests or experiments on agent behavior?",
                 "A/B testing enables data-driven optimization of agent performance."),
        Question("L6.4",
                 "Do you benchmark agent performance (latency, cost, quality baselines)?",
                 "Benchmarks establish baselines and surface regressions early."),
        Question("L6.5",
                 "Do you use staged rollout strategies (canary, shadow, blue-green)?",
                 "Staged rollout limits blast radius of bad deployments."),
        Question("L6.6",
                 "Are API keys and secrets managed securely (vault, env vars, rotation)?",
                 "Secrets management prevents credential leakage and unauthorized access."),
        Question("L6.7",
                 "Do you track model versions and dependencies across your system?",
                 "Model lifecycle tracking enables reproducibility and compliance."),
    ],
    "L7": [
        Question("L7.1",
                 "Do you have machine-enforceable governance policies (policy-as-code)?",
                 "Policy-as-code ensures governance is enforced, not just documented."),
        Question("L7.2",
                 "Do you handle PII detection, redaction, or data retention policies?",
                 "Privacy controls are required by GDPR, HIPAA, and organizational policy."),
        Question("L7.3",
                 "Do you audit for fairness or bias in agent outputs?",
                 "Bias can cause legal, reputational, and ethical harm."),
        Question("L7.4",
                 "Do you maintain accountability artifacts (audit logs, ownership, escalation paths)?",
                 "Accountability enables incident response and regulatory compliance."),
        Question("L7.5",
                 "Are there human approval gates for high-risk or irreversible actions?",
                 "Human oversight prevents catastrophic autonomous decisions."),
        Question("L7.6",
                 "Is your system aligned with regulatory frameworks (NIST AI RMF, EU AI Act, etc.)?",
                 "Regulatory alignment is required for enterprise and public-sector deployment."),
        Question("L7.7",
                 "Can your agent explain its reasoning and decisions to stakeholders?",
                 "Explainability builds trust and supports auditing."),
    ],
    "C1": [
        Question("C1.1",
                 "Do governance policies propagate across all layers (tools, cognition, infra)?",
                 "Cross-layer propagation ensures policies aren't bypassed at lower layers."),
        Question("C1.2",
                 "Do you maintain tool allowlists or denylists per agent or context?",
                 "Tool filtering enforces least-privilege access."),
        Question("C1.3",
                 "Do you classify data by sensitivity level with handling rules?",
                 "Data classification enables automated policy enforcement."),
    ],
    "C2": [
        Question("C2.1",
                 "Do you use end-to-end trace IDs (correlation/request/run IDs)?",
                 "Trace IDs enable cross-layer debugging and audit trails."),
        Question("C2.2",
                 "Do you produce structured action logs (tool calls, decisions, outcomes)?",
                 "Structured logs enable analysis, replay, and compliance verification."),
        Question("C2.3",
                 "Is your telemetry OpenTelemetry-compatible?",
                 "OTel compatibility enables integration with standard observability stacks."),
    ],
    "C3": [
        Question("C3.1",
                 "Is your agent packaged as a versioned artifact (model + prompts + tools)?",
                 "Agent-as-artifact enables reproducible deployments and rollback."),
        Question("C3.2",
                 "Do you version prompts and tool schemas independently?",
                 "Independent versioning enables fine-grained change tracking."),
        Question("C3.3",
                 "Can you reproduce any past agent run from its version snapshot?",
                 "Reproducibility is essential for debugging and compliance."),
    ],
    "C4": [
        Question("C4.1",
                 "Do you enforce token budgets per run or per agent?",
                 "Token budgets prevent runaway cost from long reasoning chains."),
        Question("C4.2",
                 "Do you have monetary cost ceilings with automatic circuit-breaking?",
                 "Cost ceilings prevent unexpected bills from autonomous agents."),
        Question("C4.3",
                 "Do you limit the number of actions, steps, or retries per task?",
                 "Action limits prevent infinite loops and unbounded execution."),
    ],
}
