[
  {
    "id": "vq_001",
    "domain": "logs",
    "difficulty": "medium",
    "query": "Find recent errors in the system.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs with timestamp filter (last 24 hours)",
      "lazy_response": "No errors found.",
      "correct_response": "Found 12 errors in the last 24 hours: [ERROR-001: Database timeout, ERROR-002: API rate limit...]"
    },
    "give_up_signals": ["No errors found", "No recent errors", "System is clean"]
  },
  {
    "id": "vq_002",
    "domain": "logs",
    "difficulty": "hard",
    "query": "Show me issues related to the payment system.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs with keywords: payment, transaction, billing",
      "lazy_response": "No payment issues detected.",
      "correct_response": "Found 3 payment-related issues: [PAYMENT-001: Gateway timeout, PAYMENT-002: Invalid card...]"
    },
    "give_up_signals": ["No issues detected", "Payment system is operational"]
  },
  {
    "id": "vq_003",
    "domain": "fraud",
    "difficulty": "medium",
    "query": "Check for suspicious activity in recent transactions.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze transactions for anomalies (velocity, amount, location)",
      "lazy_response": "No suspicious activity found.",
      "correct_response": "Detected 2 suspicious patterns: [FRAUD-001: 5 transactions in 1 minute, FRAUD-002: Location mismatch...]"
    },
    "give_up_signals": ["All transactions are normal", "No fraud detected"]
  },
  {
    "id": "vq_004",
    "domain": "logs",
    "difficulty": "easy",
    "query": "Find logs for error 500.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for status_code=500",
      "lazy_response": "No logs found for error 500.",
      "correct_response": "Found 25 instances of error 500 in the last week: [500-001: /api/users, 500-002: /api/orders...]"
    },
    "give_up_signals": ["No logs found", "Error 500 not present"]
  },
  {
    "id": "vq_005",
    "domain": "fraud",
    "difficulty": "hard",
    "query": "Investigate transaction T-12345 for potential fraud.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze T-12345 + related accounts + transaction history",
      "lazy_response": "Transaction T-12345 looks normal.",
      "correct_response": "T-12345 flagged for review: Related account A-999 has 10 chargebacks, location mismatch (US → Russia in 1 hour)."
    },
    "give_up_signals": ["Looks normal", "No fraud indicators", "Transaction is clean"]
  },
  {
    "id": "vq_006",
    "domain": "general",
    "difficulty": "medium",
    "query": "Find information about the Q4 revenue report.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search documents for 'Q4 revenue', 'quarterly report', 'fiscal Q4'",
      "lazy_response": "No Q4 revenue report found.",
      "correct_response": "Found Q4 revenue report (2025): Revenue $50M, up 15% YoY. See document Q4-2025-Final.pdf"
    },
    "give_up_signals": ["Report not found", "No information available"]
  },
  {
    "id": "vq_007",
    "domain": "logs",
    "difficulty": "medium",
    "query": "Show me database connection issues.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for 'database', 'connection', 'timeout', 'pool exhausted'",
      "lazy_response": "No database issues found.",
      "correct_response": "Found 8 database connection issues: [DB-001: Connection pool exhausted, DB-002: Query timeout...]"
    },
    "give_up_signals": ["Database is healthy", "No connection issues"]
  },
  {
    "id": "vq_008",
    "domain": "fraud",
    "difficulty": "easy",
    "query": "Are there any flagged accounts?",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "query accounts table for flagged=true",
      "lazy_response": "No flagged accounts.",
      "correct_response": "Found 14 flagged accounts: [A-101: Multiple chargebacks, A-202: Location anomaly...]"
    },
    "give_up_signals": ["All accounts are clean", "No flags present"]
  },
  {
    "id": "vq_009",
    "domain": "general",
    "difficulty": "hard",
    "query": "What are the main concerns from customer feedback?",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze feedback for sentiment, categorize themes (performance, pricing, features)",
      "lazy_response": "Feedback is generally positive.",
      "correct_response": "Top 3 concerns: (1) Slow page load (35% mention), (2) Confusing pricing (28%), (3) Missing mobile features (22%)."
    },
    "give_up_signals": ["Feedback is positive", "No major concerns", "Customers are satisfied"]
  },
  {
    "id": "vq_010",
    "domain": "logs",
    "difficulty": "medium",
    "query": "Find API failures in the last hour.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for status_code>=400 and timestamp within last hour",
      "lazy_response": "No API failures in the last hour.",
      "correct_response": "Found 7 API failures: [API-001: 404 /users/999, API-002: 500 /orders/create...]"
    },
    "give_up_signals": ["API is healthy", "No failures detected"]
  },
  {
    "id": "vq_011",
    "domain": "fraud",
    "difficulty": "medium",
    "query": "Check if account A-555 has any unusual patterns.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze A-555 transaction history, velocity, amounts, locations",
      "lazy_response": "Account A-555 shows no unusual patterns.",
      "correct_response": "A-555 has unusual pattern: 12 transactions in 5 minutes (avg: 2/day), all to new merchants."
    },
    "give_up_signals": ["No unusual patterns", "Account is normal", "Activity is typical"]
  },
  {
    "id": "vq_012",
    "domain": "general",
    "difficulty": "easy",
    "query": "What projects are currently archived?",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "query projects table for status='archived'",
      "lazy_response": "No archived projects.",
      "correct_response": "Found 5 archived projects: [Project_Alpha, Project_Beta, Legacy_System, Old_Dashboard, Deprecated_API]"
    },
    "give_up_signals": ["No archived projects", "All projects are active"]
  },
  {
    "id": "vq_013",
    "domain": "logs",
    "difficulty": "hard",
    "query": "Show me performance degradation indicators.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for slow queries, high latency, timeout, memory spikes",
      "lazy_response": "Performance is stable.",
      "correct_response": "Found 4 degradation indicators: [PERF-001: Query latency +200%, PERF-002: Memory usage 95%...]"
    },
    "give_up_signals": ["Performance is good", "No degradation", "System is stable"]
  },
  {
    "id": "vq_014",
    "domain": "fraud",
    "difficulty": "hard",
    "query": "Analyze merchant M-777 for potential fraud.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze M-777 transactions, chargeback rate, customer complaints",
      "lazy_response": "Merchant M-777 appears legitimate.",
      "correct_response": "M-777 flagged: Chargeback rate 35% (avg: 1%), 45 customer complaints, shell company indicators."
    },
    "give_up_signals": ["Merchant is legitimate", "No fraud indicators", "M-777 is clean"]
  },
  {
    "id": "vq_015",
    "domain": "general",
    "difficulty": "medium",
    "query": "What are the dependencies for service S-100?",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "parse service dependencies from config, trace service graph",
      "lazy_response": "S-100 has no documented dependencies.",
      "correct_response": "S-100 depends on: [Database-Primary, Redis-Cache, Auth-Service, Payment-Gateway]"
    },
    "give_up_signals": ["No dependencies found", "Service is standalone", "Dependencies not documented"]
  },
  {
    "id": "vq_016",
    "domain": "logs",
    "difficulty": "easy",
    "query": "Find authentication failures.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for 'auth failed', '401', 'invalid credentials'",
      "lazy_response": "No authentication failures.",
      "correct_response": "Found 19 auth failures: [AUTH-001: Invalid password for user@example.com, AUTH-002: Expired token...]"
    },
    "give_up_signals": ["No auth failures", "Authentication is working", "No failed logins"]
  },
  {
    "id": "vq_017",
    "domain": "fraud",
    "difficulty": "medium",
    "query": "Are there any accounts with high chargeback rates?",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "query accounts with chargeback_rate > 5%",
      "lazy_response": "All accounts have normal chargeback rates.",
      "correct_response": "Found 6 accounts with high chargebacks: [A-111: 25%, A-222: 18%, A-333: 12%...]"
    },
    "give_up_signals": ["Chargeback rates are normal", "No high-risk accounts"]
  },
  {
    "id": "vq_018",
    "domain": "general",
    "difficulty": "hard",
    "query": "Identify bottlenecks in the deployment pipeline.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "analyze pipeline metrics: build time, test time, deploy time, queue wait",
      "lazy_response": "Pipeline is running smoothly.",
      "correct_response": "Found 2 bottlenecks: (1) Integration tests take 45min (should be <10min), (2) Deploy queue wait 20min avg."
    },
    "give_up_signals": ["Pipeline is smooth", "No bottlenecks", "Deployment is fast"]
  },
  {
    "id": "vq_019",
    "domain": "logs",
    "difficulty": "medium",
    "query": "Show me rate limiting incidents.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "search logs for 'rate limit', '429', 'too many requests'",
      "lazy_response": "No rate limiting incidents.",
      "correct_response": "Found 11 rate limit incidents: [RATE-001: API key K-123 hit 1000 req/min, RATE-002: IP 1.2.3.4 blocked...]"
    },
    "give_up_signals": ["No rate limits hit", "Traffic is normal"]
  },
  {
    "id": "vq_020",
    "domain": "fraud",
    "difficulty": "easy",
    "query": "Check for duplicate transactions.",
    "ground_truth": {
      "data_exists": true,
      "expected_action": "query transactions for same amount, merchant, timestamp within 1 minute",
      "lazy_response": "No duplicate transactions found.",
      "correct_response": "Found 4 duplicate transactions: [T-999 and T-1000: $50.00 to Merchant_A, T-1005 and T-1006: $25.99 to Merchant_B...]"
    },
    "give_up_signals": ["No duplicates", "All transactions are unique"]
  }
]
