Coverage for src\cmvk\benchmarks.py: 0%

1"""

2 CMVK Benchmark Suite

3

4 Framework for benchmarking single-model vs multi-model verification accuracy.

5 This creates the infrastructure for benchmarks - actual results require running

6 with real LLM API calls.

7

8 Usage:

9 python -m cmvk.benchmarks.run --models gpt-4,claude-sonnet-4,gemini-pro

10"""

11

12 from __future__ import annotations

13

14 import json

15 import time

16 from dataclasses import dataclass, field

17 from datetime import datetime

18 from enum import Enum

19 from pathlib import Path

20 from typing import Any, Callable, Optional

21

22 import numpy as np

23

24

25 class TaskCategory(Enum):

26 """Categories of benchmark tasks."""

27 FACTUAL = "factual" # Verifiable facts

28 MATHEMATICAL = "mathematical" # Numeric calculations

29 REASONING = "reasoning" # Logic and inference

30 EXTRACTION = "extraction" # Information extraction

31

32

33@dataclass

34 class BenchmarkTask:

35 """A single benchmark task."""

36 id: str

37 category: TaskCategory

38 prompt: str

39 ground_truth: Any

40 difficulty: str = "medium" # easy, medium, hard

41 metadata: dict = field(default_factory=dict)

42

43

44@dataclass

45 class ModelResponse:

46 """Response from a single model."""

47 model_name: str

48 response: Any

49 latency_ms: float

50 raw_output: str = ""

51

52

53@dataclass

54 class VerificationResult:

55 """Result of verifying a task."""

56 task_id: str

57 is_correct: bool

58 confidence: float

59 responses: list[ModelResponse]

60 consensus_method: str

61 drift_score: float = 0.0

62 details: dict = field(default_factory=dict)

63

64

65@dataclass

66 class BenchmarkResults:

67 """Aggregate benchmark results."""

68 total_tasks: int

69 correct: int

70 accuracy: float

71 avg_latency_ms: float

72 by_category: dict[str, dict]

73 by_difficulty: dict[str, dict]

74 timestamp: str

75 config: dict

76

77

78 class ConsensusMethod(Enum):

79 """Methods for reaching consensus across models."""

80 MAJORITY_VOTE = "majority_vote"

81 UNANIMOUS = "unanimous"

82 WEIGHTED = "weighted"

83 DRIFT_THRESHOLD = "drift_threshold"

84

85

86 class CMVKBenchmark:

87 """

88 Benchmark framework for CMVK verification.

89

90 This class provides infrastructure for running benchmarks.

91 Actual model calls must be provided via the `model_fn` parameter.

92 """

93

94 def __init__(

95 self,

96 models: list[str],

97 model_fn: Optional[Callable[[str, str], str]] = None,

98 consensus_method: ConsensusMethod = ConsensusMethod.DRIFT_THRESHOLD,

99 drift_threshold: float = 0.15

100 ):

101 """

102 Initialize benchmark.

103

104 Args:

105 models: List of model names to use

106 model_fn: Function(model_name, prompt) -> response

107 If None, uses mock responses for testing

108 consensus_method: How to combine model responses

109 drift_threshold: Threshold for drift-based consensus

110 """

111 self.models = models

112 self.model_fn = model_fn or self._mock_model_fn

113 self.consensus_method = consensus_method

114 self.drift_threshold = drift_threshold

115 self.results: list[VerificationResult] = []

116

117 def _mock_model_fn(self, model_name: str, prompt: str) -> str:

118 """Mock model function for testing the framework."""

119 # Return deterministic mock response based on prompt hash

120 return f"Mock response from {model_name} for prompt hash {hash(prompt) % 1000}"

121

122 def load_tasks(self, path: Path) -> list[BenchmarkTask]:

123 """Load benchmark tasks from JSON file."""

124 with open(path) as f:

125 data = json.load(f)

126

127 return [

128 BenchmarkTask(

129 id=t["id"],

130 category=TaskCategory(t["category"]),

131 prompt=t["prompt"],

132 ground_truth=t["ground_truth"],

133 difficulty=t.get("difficulty", "medium"),

134 metadata=t.get("metadata", {})

135 )

136 for t in data["tasks"]

137 ]

138

139 def run_single_model(self, tasks: list[BenchmarkTask], model: str) -> list[VerificationResult]:

140 """Run benchmark with a single model."""

141 results = []

142

143 for task in tasks:

144 start = time.perf_counter()

145 response = self.model_fn(model, task.prompt)

146 latency = (time.perf_counter() - start) * 1000

147

148 model_response = ModelResponse(

149 model_name=model,

150 response=response,

151 latency_ms=latency,

152 raw_output=response

153 )

154

155 is_correct = self._check_correctness(response, task.ground_truth)

156

157 results.append(VerificationResult(

158 task_id=task.id,

159 is_correct=is_correct,

160 confidence=1.0 if is_correct else 0.0,

161 responses=[model_response],

162 consensus_method="single_model"

163 ))

164

165 return results

166

167 def run_multi_model(self, tasks: list[BenchmarkTask]) -> list[VerificationResult]:

168 """Run benchmark with multiple models using CMVK consensus."""

169 results = []

170

171 for task in tasks:

172 responses = []

173

174 # Get response from each model

175 for model in self.models:

176 start = time.perf_counter()

177 response = self.model_fn(model, task.prompt)

178 latency = (time.perf_counter() - start) * 1000

179

180 responses.append(ModelResponse(

181 model_name=model,

182 response=response,

183 latency_ms=latency,

184 raw_output=response

185 ))

186

187 # Apply consensus method

188 consensus_result = self._apply_consensus(responses, task.ground_truth)

189

190 results.append(VerificationResult(

191 task_id=task.id,

192 is_correct=consensus_result["is_correct"],

193 confidence=consensus_result["confidence"],

194 responses=responses,

195 consensus_method=self.consensus_method.value,

196 drift_score=consensus_result.get("drift_score", 0.0),

197 details=consensus_result.get("details", {})

198 ))

199

200 return results

201

202 def _apply_consensus(self, responses: list[ModelResponse], ground_truth: Any) -> dict:

203 """

204 Apply consensus method to multiple model responses.

205

206 This is where the CMVK algorithm is applied.

207 """

208 if self.consensus_method == ConsensusMethod.MAJORITY_VOTE:

209 return self._majority_vote_consensus(responses, ground_truth)

210 elif self.consensus_method == ConsensusMethod.UNANIMOUS:

211 return self._unanimous_consensus(responses, ground_truth)

212 elif self.consensus_method == ConsensusMethod.DRIFT_THRESHOLD:

213 return self._drift_consensus(responses, ground_truth)

214 else:

215 return self._majority_vote_consensus(responses, ground_truth)

216

217 def _majority_vote_consensus(self, responses: list[ModelResponse], ground_truth: Any) -> dict:

218 """Simple majority voting."""

219 correct_count = sum(

220 1 for r in responses

221 if self._check_correctness(r.response, ground_truth)

222 )

223

224 majority = correct_count > len(responses) / 2

225 confidence = correct_count / len(responses)

226

227 return {

228 "is_correct": majority,

229 "confidence": confidence,

230 "details": {"correct_count": correct_count, "total": len(responses)}

231 }

232

233 def _unanimous_consensus(self, responses: list[ModelResponse], ground_truth: Any) -> dict:

234 """Require all models to agree."""

235 all_correct = all(

236 self._check_correctness(r.response, ground_truth)

237 for r in responses

238 )

239

240 return {

241 "is_correct": all_correct,

242 "confidence": 1.0 if all_correct else 0.0,

243 "details": {"unanimous": all_correct}

244 }

245

246 def _drift_consensus(self, responses: list[ModelResponse], ground_truth: Any) -> dict:

247 """

248 CMVK drift-based consensus.

249

250 Algorithm:

251 1. Convert responses to vectors (embeddings or numeric)

252 2. Calculate pairwise drift between all responses

253 3. If max drift > threshold, responses disagree → flag for review

254 4. If responses agree, check against ground truth

255 """

256 # Calculate drift between responses

257 drift_scores = self._calculate_pairwise_drift(responses)

258 max_drift = max(drift_scores) if drift_scores else 0.0

259 avg_drift = np.mean(drift_scores) if drift_scores else 0.0

260

261 # High drift = disagreement = low confidence

262 if max_drift > self.drift_threshold:

263 # Models disagree significantly

264 return {

265 "is_correct": False, # Can't trust when models disagree

266 "confidence": 1.0 - max_drift,

267 "drift_score": max_drift,

268 "details": {

269 "disagreement_detected": True,

270 "max_drift": max_drift,

271 "avg_drift": avg_drift,

272 "threshold": self.drift_threshold

273 }

274 }

275

276 # Models agree - check correctness

277 # Use first response as representative (they all agree)

278 is_correct = self._check_correctness(responses[0].response, ground_truth)

279

280 return {

281 "is_correct": is_correct,

282 "confidence": 1.0 - avg_drift,

283 "drift_score": avg_drift,

284 "details": {

285 "disagreement_detected": False,

286 "max_drift": max_drift,

287 "avg_drift": avg_drift,

288 "consensus_response": responses[0].response

289 }

290 }

291

292 def _calculate_pairwise_drift(self, responses: list[ModelResponse]) -> list[float]:

293 """Calculate drift between all pairs of responses."""

294 drifts = []

295

296 for i in range(len(responses)):

297 for j in range(i + 1, len(responses)):

298 drift = self._response_drift(

299 responses[i].response,

300 responses[j].response

301 )

302 drifts.append(drift)

303

304 return drifts

305

306 def _response_drift(self, response_a: Any, response_b: Any) -> float:

307 """

308 Calculate drift between two responses.

309

310 For numeric responses: normalized absolute difference

311 For string responses: Levenshtein-based similarity

312 For structured: recursive comparison

313 """

314 # Handle numeric

315 if isinstance(response_a, (int, float)) and isinstance(response_b, (int, float)):

316 max_val = max(abs(response_a), abs(response_b), 1)

317 return abs(response_a - response_b) / max_val

318

319 # Handle string

320 if isinstance(response_a, str) and isinstance(response_b, str):

321 return self._string_drift(response_a, response_b)

322

323 # Handle lists/arrays

324 if isinstance(response_a, (list, np.ndarray)) and isinstance(response_b, (list, np.ndarray)):

325 a = np.array(response_a)

326 b = np.array(response_b)

327 if a.shape == b.shape:

328 return float(np.linalg.norm(a - b) / (np.linalg.norm(a) + np.linalg.norm(b) + 1e-10))

329

330 # Fallback: exact match

331 return 0.0 if response_a == response_b else 1.0

332

333 def _string_drift(self, a: str, b: str) -> float:

334 """Calculate drift between strings using character-level comparison."""

335 if a == b:

336 return 0.0

337 if not a or not b:

338 return 1.0

339

340 # Simple normalized edit distance approximation

341 # (For production, use proper Levenshtein or embedding similarity)

342 common = sum(1 for c in a if c in b)

343 total = len(a) + len(b)

344 similarity = (2 * common) / total if total > 0 else 0

345 return 1.0 - similarity

346

347 def _check_correctness(self, response: Any, ground_truth: Any) -> bool:

348 """Check if response matches ground truth."""

349 # Exact match

350 if response == ground_truth:

351 return True

352

353 # Numeric tolerance

354 if isinstance(response, (int, float)) and isinstance(ground_truth, (int, float)):

355 return abs(response - ground_truth) < 0.01 * abs(ground_truth + 1e-10)

356

357 # String containment (ground truth in response)

358 if isinstance(response, str) and isinstance(ground_truth, str):

359 return ground_truth.lower() in response.lower()

360

361 return False

362

363 def aggregate_results(self, results: list[VerificationResult], config: dict = None) -> BenchmarkResults:

364 """Aggregate results into summary statistics."""

365 correct = sum(1 for r in results if r.is_correct)

366 total = len(results)

367

368 # Group by category

369 by_category = {}

370 for r in results:

371 # Would need task info to group properly

372 pass

373

374 # Calculate latency

375 all_latencies = []

376 for r in results:

377 for resp in r.responses:

378 all_latencies.append(resp.latency_ms)

379

380 avg_latency = np.mean(all_latencies) if all_latencies else 0.0

381

382 return BenchmarkResults(

383 total_tasks=total,

384 correct=correct,

385 accuracy=correct / total if total > 0 else 0.0,

386 avg_latency_ms=avg_latency,

387 by_category={},

388 by_difficulty={},

389 timestamp=datetime.utcnow().isoformat(),

390 config=config or {}

391 )

392

393 def save_results(self, results: BenchmarkResults, path: Path) -> None:

394 """Save results to JSON file."""

395 with open(path, "w") as f:

396 json.dump({

397 "total_tasks": results.total_tasks,

398 "correct": results.correct,

399 "accuracy": results.accuracy,

400 "avg_latency_ms": results.avg_latency_ms,

401 "by_category": results.by_category,

402 "by_difficulty": results.by_difficulty,

403 "timestamp": results.timestamp,

404 "config": results.config

405 }, f, indent=2)

406

407

408 def create_sample_tasks(n: int = 100) -> list[BenchmarkTask]:

409 """Create sample benchmark tasks for testing."""

410 tasks = []

411

412 categories = list(TaskCategory)

413 difficulties = ["easy", "medium", "hard"]

414

415 for i in range(n):

416 cat = categories[i % len(categories)]

417 diff = difficulties[i % len(difficulties)]

418

419 if cat == TaskCategory.FACTUAL:

420 tasks.append(BenchmarkTask(

421 id=f"factual_{i}",

422 category=cat,

423 prompt=f"What is the capital of country {i % 50}?",

424 ground_truth=f"Capital_{i % 50}",

425 difficulty=diff

426 ))

427 elif cat == TaskCategory.MATHEMATICAL:

428 a, b = i * 7, i * 3

429 tasks.append(BenchmarkTask(

430 id=f"math_{i}",

431 category=cat,

432 prompt=f"What is {a} + {b}?",

433 ground_truth=a + b,

434 difficulty=diff

435 ))

436 elif cat == TaskCategory.REASONING:

437 tasks.append(BenchmarkTask(

438 id=f"reasoning_{i}",

439 category=cat,

440 prompt=f"If A implies B, and A is true, what can we conclude about B?",

441 ground_truth="B is true",

442 difficulty=diff

443 ))

444 else:

445 tasks.append(BenchmarkTask(

446 id=f"extraction_{i}",

447 category=cat,

448 prompt=f"Extract the number from: 'There are {i * 5} items'",

449 ground_truth=i * 5,

450 difficulty=diff

451 ))

452

453 return tasks

454

455

456 if __name__ == "__main__":

457 # Demo: Run benchmark with mock data

458 benchmark = CMVKBenchmark(

459 models=["model_a", "model_b", "model_c"],

460 consensus_method=ConsensusMethod.DRIFT_THRESHOLD,

461 drift_threshold=0.15

462 )

463

464 tasks = create_sample_tasks(100)

465

466 print("Running single-model benchmark...")

467 single_results = benchmark.run_single_model(tasks, "model_a")

468 single_summary = benchmark.aggregate_results(single_results, {"mode": "single", "model": "model_a"})

469 print(f"Single model accuracy: {single_summary.accuracy:.2%}")

470

471 print("\nRunning multi-model (CMVK) benchmark...")

472 multi_results = benchmark.run_multi_model(tasks)

473 multi_summary = benchmark.aggregate_results(multi_results, {"mode": "cmvk", "models": benchmark.models})

474 print(f"CMVK accuracy: {multi_summary.accuracy:.2%}")

475

476 print("\nNote: These are mock results. Run with real LLM APIs for actual benchmarks.")