# Agent OS Prometheus Alert Rules
# Import into Prometheus AlertManager

groups:
  - name: agent-os-safety
    rules:
      # =========================================================================
      # CRITICAL: Safety Violations (Page immediately)
      # =========================================================================
      
      - alert: AgentOSHighViolationRate
        expr: agent_os_violation_rate{window="all_time"} > 0.01
        for: 1m
        labels:
          severity: critical
          team: ai-safety
        annotations:
          summary: "High policy violation rate detected"
          description: |
            Violation rate is {{ $value | humanizePercentage }} (threshold: 1%).
            This exceeds the enterprise SLA. Investigate immediately.
          runbook_url: "https://docs.agent-os.dev/runbooks/high-violation-rate"
      
      - alert: AgentOSSIGKILLSpike
        expr: increase(agent_os_sigkill_total[5m]) > 5
        for: 0m
        labels:
          severity: critical
          team: ai-safety
        annotations:
          summary: "SIGKILL spike detected"
          description: |
            {{ $value }} SIGKILL signals issued in the last 5 minutes.
            This indicates repeated policy violations. Agent {{ $labels.agent_id }} may be compromised.
          runbook_url: "https://docs.agent-os.dev/runbooks/sigkill-spike"
      
      - alert: AgentOSKernelCrash
        expr: increase(agent_os_kernel_crashes_total[5m]) > 0
        for: 0m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "KERNEL PANIC - Agent OS kernel crashed"
          description: |
            The Agent OS kernel crashed. This should NEVER happen.
            All agents are unprotected until recovery. Escalate immediately.
          runbook_url: "https://docs.agent-os.dev/runbooks/kernel-panic"

  - name: agent-os-performance
    rules:
      # =========================================================================
      # WARNING: Performance Degradation
      # =========================================================================
      
      - alert: AgentOSHighPolicyLatency
        expr: histogram_quantile(0.99, rate(agent_os_policy_check_duration_seconds_bucket[5m])) > 0.01
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Policy check latency exceeds SLA"
          description: |
            p99 policy check latency is {{ $value | humanizeDuration }} (threshold: 10ms).
            This may impact agent responsiveness.
          runbook_url: "https://docs.agent-os.dev/runbooks/high-latency"
      
      - alert: AgentOSExecutionTimeout
        expr: histogram_quantile(0.99, rate(agent_os_execution_duration_seconds_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Agent execution times increasing"
          description: |
            p99 execution time is {{ $value | humanizeDuration }}.
            Agents may be stuck or overloaded.

  - name: agent-os-availability
    rules:
      # =========================================================================
      # WARNING: Availability Issues
      # =========================================================================
      
      - alert: AgentOSKernelDown
        expr: up{job="agent-os"} == 0
        for: 1m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Agent OS kernel is down"
          description: |
            The Agent OS kernel is not responding to health checks.
            Agents are running without governance.
      
      - alert: AgentOSHighMTTR
        expr: histogram_quantile(0.50, rate(agent_os_mttr_seconds_bucket[1h])) > 60
        for: 15m
        labels:
          severity: warning
          team: sre
        annotations:
          summary: "Recovery time is too slow"
          description: |
            Median time to recover after SIGKILL is {{ $value | humanizeDuration }}.
            Target is <60 seconds.

  - name: agent-os-cmvk
    rules:
      # =========================================================================
      # CMVK: Model Verification Alerts
      # =========================================================================
      
      - alert: CMVKLowConsensus
        expr: agent_os_cmvk_consensus_ratio < 0.8
        for: 10m
        labels:
          severity: warning
          team: ml-ops
        annotations:
          summary: "CMVK model consensus is low"
          description: |
            Model consensus ratio is {{ $value | humanizePercentage }} (threshold: 80%).
            Models are disagreeing more than expected. Investigate:
            - Model drift between versions
            - Training data issues
            - Ambiguous claims being submitted
          runbook_url: "https://docs.agent-os.dev/runbooks/cmvk-low-consensus"
      
      - alert: CMVKHighDrift
        expr: histogram_quantile(0.95, rate(agent_os_cmvk_drift_score_bucket[15m])) > 0.25
        for: 10m
        labels:
          severity: warning
          team: ml-ops
        annotations:
          summary: "High drift between verification models"
          description: |
            95th percentile drift score is {{ $value }} (threshold: 0.25).
            Models are producing significantly different outputs.
      
      - alert: CMVKVerificationSlow
        expr: histogram_quantile(0.95, rate(agent_os_cmvk_verification_duration_seconds_bucket[5m])) > 10
        for: 5m
        labels:
          severity: warning
          team: ml-ops
        annotations:
          summary: "CMVK verification taking too long"
          description: |
            p95 verification time is {{ $value | humanizeDuration }}.
            This impacts user experience and may indicate model API issues.
      
      - alert: CMVKHighDisagreementRate
        expr: rate(agent_os_cmvk_model_disagreements_total[1h]) > 10
        for: 30m
        labels:
          severity: info
          team: ml-ops
        annotations:
          summary: "Elevated model disagreement rate"
          description: |
            Model pair {{ $labels.model_pair }} has {{ $value | humanize }} disagreements/hour.
            Review if this affects verification quality.

  - name: agent-os-agents
    rules:
      # =========================================================================
      # Agent Health Alerts
      # =========================================================================
      
      - alert: AgentCrashLoop
        expr: increase(agent_os_agent_crashes_total[10m]) > 3
        for: 0m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Agent crash loop detected"
          description: |
            Agent {{ $labels.agent_id }} has crashed {{ $value }} times in 10 minutes.
            Reason: {{ $labels.reason }}
      
      - alert: AgentHighErrorRate
        expr: rate(agent_os_agent_errors_total[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Agent experiencing high error rate"
          description: |
            Agent {{ $labels.agent_id }} has error rate of {{ $value | humanize }}/sec.
            Error type: {{ $labels.error_type }}
