Coverage for src\cmvk\verification.py: 13%

1"""

2 CMVK Verification Module - Pure Mathematical Functions

3

4 This module provides pure functions for calculating drift/hallucination scores

5 between two outputs. These functions have no side effects and use only

6 numpy/scipy for mathematical operations.

7

8 Layer 1: The Primitive - Mathematical and adversarial verification.

9

10 Enhanced Features (v0.2.0):

11 - Configurable distance metrics (cosine, euclidean, manhattan, etc.)

12 - Dimensional weighting for importance-based drift calculation

13 - Threshold profiles for domain-specific verification

14 - Explainable drift with per-dimension contributions

15 - Batch verification for efficiency

16 - Audit trail integration

17"""

18

19 from __future__ import annotations

20

21 from collections.abc import Sequence

22 from dataclasses import dataclass

23 from enum import Enum

24 from typing import TYPE_CHECKING, Any

25

26 import numpy as np

27 from numpy.typing import ArrayLike

28

29 try:

30 from scipy import stats

31

32 HAS_SCIPY = True

33 except ImportError:

34 HAS_SCIPY = False

35

36 if TYPE_CHECKING:

37 from .audit import AuditTrail

38

39

40 class DriftType(Enum):

41 """Types of drift/divergence detected between outputs."""

42

43 SEMANTIC = "semantic"

44 STRUCTURAL = "structural"

45 NUMERICAL = "numerical"

46 LEXICAL = "lexical"

47

48

49@dataclass(frozen=True)

50 class VerificationScore:

51 """

52 Immutable result of verification between two outputs.

53

54 Attributes:

55 drift_score: Overall drift score between 0.0 (identical) and 1.0 (completely different)

56 confidence: Confidence in the score (0.0 to 1.0)

57 drift_type: Primary type of drift detected

58 details: Dictionary with component scores

59 explanation: Optional drift explanation with dimension contributions (CMVK-010)

60 """

61

62 drift_score: float

63 confidence: float

64 drift_type: DriftType

65 details: dict

66 explanation: dict | None = None

67

68 def passed(self, threshold: float = 0.3) -> bool:

69 """Check if drift is within acceptable threshold."""

70 return self.drift_score <= threshold

71

72 def to_dict(self) -> dict:

73 """Convert to dictionary for serialization."""

74 return {

75 "drift_score": self.drift_score,

76 "confidence": self.confidence,

77 "drift_type": self.drift_type.value,

78 "details": self.details,

79 "explanation": self.explanation,

80 }

81

82

83@dataclass(frozen=True)

84 class DriftExplanation:

85 """

86 Detailed explanation of drift between two vectors (CMVK-010).

87

88 Attributes:

89 primary_drift_dimension: Index or name of dimension with highest contribution

90 dimension_contributions: Mapping of dimension to its contribution percentage

91 top_contributors: List of top N contributing dimensions

92 metric_used: The distance metric used

93 interpretation: Human-readable interpretation of the drift

94 """

95

96 primary_drift_dimension: str | int

97 dimension_contributions: dict[str | int, float]

98 top_contributors: list[tuple[str | int, float]]

99 metric_used: str

100 interpretation: str

101

102 def to_dict(self) -> dict:

103 """Convert to dictionary."""

104 return {

105 "primary_drift_dimension": self.primary_drift_dimension,

106 "dimension_contributions": self.dimension_contributions,

107 "top_contributors": self.top_contributors,

108 "metric_used": self.metric_used,

109 "interpretation": self.interpretation,

110 }

111

112

113 def verify(output_a: str, output_b: str) -> VerificationScore:

114 """

115 Calculate drift/hallucination score between two outputs.

116

117 This is the primary verification function - a pure function with no side effects.

118 Takes two outputs and returns a score indicating their divergence.

119

120 Args:

121 output_a: First output (typically from model A / generator)

122 output_b: Second output (typically from model B / verifier)

123

124 Returns:

125 VerificationScore with drift score, confidence, and details

126

127 Example:

128 >>> score = verify("def add(a, b): return a + b", "def add(x, y): return x + y")

129 >>> score.drift_score # Low score - semantically similar

130 0.15

131 """

132 if not output_a and not output_b:

133 return VerificationScore(

134 drift_score=0.0,

135 confidence=1.0,

136 drift_type=DriftType.LEXICAL,

137 details={"reason": "both_empty"},

138 )

139

140 if not output_a or not output_b:

141 return VerificationScore(

142 drift_score=1.0,

143 confidence=1.0,

144 drift_type=DriftType.STRUCTURAL,

145 details={"reason": "one_empty"},

146 )

147

148 # Calculate multiple drift components

149 lexical_drift = _lexical_drift(output_a, output_b)

150 structural_drift = _structural_drift(output_a, output_b)

151 numerical_drift = _numerical_drift(output_a, output_b)

152

153 # Weighted combination

154 weights = {"lexical": 0.3, "structural": 0.4, "numerical": 0.3}

155

156 combined_drift = (

157 weights["lexical"] * lexical_drift["score"]

158 + weights["structural"] * structural_drift["score"]

159 + weights["numerical"] * numerical_drift["score"]

160 )

161

162 # Determine primary drift type

163 scores = {

164 DriftType.LEXICAL: lexical_drift["score"],

165 DriftType.STRUCTURAL: structural_drift["score"],

166 DriftType.NUMERICAL: numerical_drift["score"],

167 }

168 primary_drift = max(scores, key=lambda k: scores[k])

169

170 # Calculate confidence based on agreement between methods

171 score_values = list(scores.values())

172 confidence = 1.0 - np.std(score_values) if len(score_values) > 1 else 0.8

173

174 return VerificationScore(

175 drift_score=float(np.clip(combined_drift, 0.0, 1.0)),

176 confidence=float(np.clip(confidence, 0.0, 1.0)),

177 drift_type=primary_drift,

178 details={

179 "lexical": lexical_drift,

180 "structural": structural_drift,

181 "numerical": numerical_drift,

182 "weights": weights,

183 },

184 )

185

186

187 def verify_embeddings(

188 embedding_a: ArrayLike,

189 embedding_b: ArrayLike,

190 metric: str = "cosine",

191 weights: ArrayLike | None = None,

192 threshold_profile: str | None = None,

193 explain: bool = False,

194 dimension_names: list[str] | None = None,

195 audit_trail: AuditTrail | None = None,

196) -> VerificationScore:

197 """

198 Calculate drift score between two embedding vectors.

199

200 Enhanced verification function with configurable metrics, weighting,

201 threshold profiles, and explainability (CMVK-001 through CMVK-010).

202

203 Args:

204 embedding_a: Embedding vector for output A (e.g., claimed values)

205 embedding_b: Embedding vector for output B (e.g., observed values)

206 metric: Distance metric to use. Options:

207 - "cosine": Cosine distance (default, normalizes vectors)

208 - "euclidean": Euclidean distance (preserves magnitude - CMVK-001)

209 - "manhattan": Manhattan/L1 distance

210 - "chebyshev": Maximum absolute difference

211 - "mahalanobis": Mahalanobis distance

212 weights: Optional weights for each dimension (CMVK-008).

213 Higher weights increase that dimension's contribution to drift.

214 threshold_profile: Name of threshold profile to use (CMVK-005).

215 Options: "carbon", "financial", "medical", "general", "strict"

216 explain: If True, include detailed drift explanation (CMVK-010)

217 dimension_names: Optional names for dimensions (for explainability)

218 audit_trail: Optional AuditTrail instance for logging (CMVK-006)

219

220 Returns:

221 VerificationScore with drift score, confidence, and optional explanation

222

223 Example:

224 >>> # Basic usage

225 >>> score = verify_embeddings(claim_vec, obs_vec)

226

227 >>> # With Euclidean distance for magnitude-sensitive comparison

228 >>> score = verify_embeddings(

229 ... claim_vec, obs_vec,

230 ... metric="euclidean",

231 ... threshold_profile="carbon",

232 ... explain=True

233 ... )

234

235 >>> # With dimensional weighting

236 >>> score = verify_embeddings(

237 ... claim_vec, obs_vec,

238 ... metric="euclidean",

239 ... weights=[0.6, 0.4], # NDVI more important than carbon

240 ... explain=True,

241 ... dimension_names=["ndvi", "carbon_stock"]

242 ... )

243 """

244 from .metrics import calculate_distance, calculate_weighted_distance

245

246 vec_a = np.asarray(embedding_a, dtype=np.float64)

247 vec_b = np.asarray(embedding_b, dtype=np.float64)

248

249 # Load threshold profile if specified

250 profile = None

251 if threshold_profile:

252 from .profiles import get_profile

253

254 profile = get_profile(threshold_profile)

255 # Use profile's default metric if none specified

256 if metric == "cosine" and profile.default_metric != "cosine":

257 metric = profile.default_metric

258

259 # Shape validation

260 if vec_a.shape != vec_b.shape:

261 result = VerificationScore(

262 drift_score=1.0,

263 confidence=0.5,

264 drift_type=DriftType.STRUCTURAL,

265 details={"reason": "shape_mismatch", "shape_a": vec_a.shape, "shape_b": vec_b.shape},

266 )

267 if audit_trail:

268 _log_to_audit(audit_trail, vec_a, vec_b, result, metric, threshold_profile)

269 return result

270

271 # Calculate distance with appropriate function

272 if weights is not None:

273 metric_result = calculate_weighted_distance(vec_a, vec_b, weights=weights, metric=metric)

274 else:

275 metric_result = calculate_distance(vec_a, vec_b, metric=metric)

276

277 # Build drift score from normalized distance

278 drift_score = float(np.clip(metric_result.normalized, 0.0, 1.0))

279

280 # Calculate confidence based on vector properties

281 confidence = _calculate_embedding_confidence(vec_a, vec_b)

282

283 # Build explanation if requested

284 explanation_dict = None

285 if explain:

286 explanation = _build_drift_explanation(

287 vec_a, vec_b, metric_result, weights, dimension_names

288 )

289 explanation_dict = explanation.to_dict()

290

291 # Build details

292 details = {

293 "metric": metric,

294 "raw_distance": metric_result.distance,

295 "normalized_distance": metric_result.normalized,

296 **metric_result.details,

297 }

298

299 # Add profile info if used

300 if profile:

301 passed = profile.is_within_threshold(drift_score, confidence)

302 severity = profile.get_severity(drift_score)

303 details["profile"] = {

304 "name": profile.name,

305 "drift_threshold": profile.drift_threshold,

306 "passed": passed,

307 "severity": severity,

308 }

309

310 result = VerificationScore(

311 drift_score=drift_score,

312 confidence=confidence,

313 drift_type=DriftType.SEMANTIC,

314 details=details,

315 explanation=explanation_dict,

316 )

317

318 # Log to audit trail if provided

319 if audit_trail:

320 _log_to_audit(audit_trail, vec_a, vec_b, result, metric, threshold_profile)

321

322 return result

323

324

325 def verify_embeddings_batch(

326 embeddings_a: Sequence[ArrayLike],

327 embeddings_b: Sequence[ArrayLike],

328 metric: str = "cosine",

329 weights: ArrayLike | None = None,

330 threshold_profile: str | None = None,

331 explain: bool = False,

332 dimension_names: list[str] | None = None,

333 audit_trail: AuditTrail | None = None,

334) -> list[VerificationScore]:

335 """

336 Verify multiple embedding pairs efficiently (CMVK-004).

337

338 Processes all pairs with consistent settings and optional audit logging.

339

340 Args:

341 embeddings_a: Sequence of embedding vectors from source A

342 embeddings_b: Sequence of embedding vectors from source B

343 metric: Distance metric (applied to all pairs)

344 weights: Dimensional weights (applied to all pairs)

345 threshold_profile: Threshold profile name

346 explain: Whether to include explanations

347 dimension_names: Optional dimension names for explainability

348 audit_trail: Optional AuditTrail for logging

349

350 Returns:

351 List of VerificationScore for each pair

352

353 Raises:

354 ValueError: If sequence lengths don't match

355 """

356 if len(embeddings_a) != len(embeddings_b):

357 raise ValueError(

358 f"Length mismatch: embeddings_a has {len(embeddings_a)} items, "

359 f"embeddings_b has {len(embeddings_b)} items"

360 )

361

362 results = []

363 for vec_a, vec_b in zip(embeddings_a, embeddings_b, strict=True):

364 score = verify_embeddings(

365 vec_a,

366 vec_b,

367 metric=metric,

368 weights=weights,

369 threshold_profile=threshold_profile,

370 explain=explain,

371 dimension_names=dimension_names,

372 audit_trail=audit_trail,

373 )

374 results.append(score)

375

376 return results

377

378

379 def aggregate_embedding_scores(

380 scores: Sequence[VerificationScore],

381 threshold_profile: str | None = None,

382) -> dict[str, Any]:

383 """

384 Aggregate multiple embedding verification scores with profile context.

385

386 Args:

387 scores: Sequence of VerificationScore objects

388 threshold_profile: Optional profile for pass/fail classification

389

390 Returns:

391 Dictionary with aggregate statistics and pass rates

392 """

393 if not scores:

394 return {"count": 0}

395

396 profile = None

397 if threshold_profile:

398 from .profiles import get_profile

399

400 profile = get_profile(threshold_profile)

401

402 drift_values = [s.drift_score for s in scores]

403 confidence_values = [s.confidence for s in scores]

404

405 # Calculate pass/fail if profile available

406 if profile:

407 passed_count = sum(

408 1 for s in scores if profile.is_within_threshold(s.drift_score, s.confidence)

409 )

410 severity_counts: dict[str, int] = {

411 "pass": 0,

412 "warning": 0,

413 "critical": 0,

414 "severe": 0,

415 }

416 for s in scores:

417 severity = profile.get_severity(s.drift_score)

418 severity_counts[severity] += 1

419 else:

420 passed_count = sum(1 for s in scores if s.drift_score <= 0.3)

421 severity_counts = {}

422

423 result: dict[str, Any] = {

424 "count": len(scores),

425 "passed_count": passed_count,

426 "failed_count": len(scores) - passed_count,

427 "pass_rate": passed_count / len(scores),

428 "mean_drift": float(np.mean(drift_values)),

429 "std_drift": float(np.std(drift_values)),

430 "min_drift": float(np.min(drift_values)),

431 "max_drift": float(np.max(drift_values)),

432 "median_drift": float(np.median(drift_values)),

433 "mean_confidence": float(np.mean(confidence_values)),

434 "p95_drift": float(np.percentile(drift_values, 95)),

435 }

436

437 if severity_counts and profile:

438 result["severity_distribution"] = severity_counts

439 result["profile_used"] = profile.name

440

441 return result

442

443

444# ============================================================================

445# Explainability Functions (CMVK-010)

446# ============================================================================

447

448

449 def _build_drift_explanation(

450 vec_a: np.ndarray,

451 vec_b: np.ndarray,

452 metric_result: Any,

453 weights: ArrayLike | None,

454 dimension_names: list[str] | None,

455) -> DriftExplanation:

456 """Build detailed drift explanation."""

457 diff = np.abs(vec_a - vec_b)

458

459 # Apply weights if provided

460 if weights is not None:

461 weights_arr = np.asarray(weights, dtype=np.float64)

462 weighted_diff = diff * weights_arr

463 else:

464 weighted_diff = diff

465

466 # Calculate per-dimension contributions

467 total_diff = np.sum(weighted_diff)

468 contributions = weighted_diff / total_diff if total_diff > 0 else np.zeros_like(diff)

469

470 # Map contributions to names or indices

471 contrib_dict: dict[str | int, float]

472 sorted_contribs: list[tuple[str | int, float]]

473 primary_dim: str | int

474

475 if dimension_names and len(dimension_names) == len(contributions):

476 contrib_dict = {

477 name: float(c) for name, c in zip(dimension_names, contributions, strict=False)

478 }

479 sorted_contribs = sorted(contrib_dict.items(), key=lambda x: x[1], reverse=True)

480 primary_dim = sorted_contribs[0][0]

481 else:

482 contrib_dict = {i: float(c) for i, c in enumerate(contributions)}

483 sorted_contribs = sorted(contrib_dict.items(), key=lambda x: x[1], reverse=True)

484 primary_dim = sorted_contribs[0][0]

485

486 # Top contributors (up to 5)

487 top_contributors: list[tuple[str | int, float]] = sorted_contribs[:5]

488

489 # Generate interpretation

490 interpretation = _generate_interpretation(

491 vec_a, vec_b, primary_dim, top_contributors, dimension_names

492 )

493

494 return DriftExplanation(

495 primary_drift_dimension=primary_dim,

496 dimension_contributions=contrib_dict,

497 top_contributors=top_contributors,

498 metric_used=metric_result.metric.value,

499 interpretation=interpretation,

500 )

501

502

503 def _generate_interpretation(

504 vec_a: np.ndarray,

505 vec_b: np.ndarray,

506 primary_dim: str | int,

507 top_contributors: list[tuple[str | int, float]],

508 dimension_names: list[str] | None,

509) -> str:

510 """Generate human-readable interpretation of drift."""

511 # Get primary dimension index

512 if isinstance(primary_dim, str) and dimension_names:

513 idx = dimension_names.index(primary_dim)

514 else:

515 idx = primary_dim if isinstance(primary_dim, int) else 0

516

517 diff_value = abs(vec_a[idx] - vec_b[idx])

518 pct_diff = (diff_value / abs(vec_a[idx])) * 100 if vec_a[idx] != 0 else float("inf")

519

520 dim_name = primary_dim if isinstance(primary_dim, str) else f"dimension {primary_dim}"

521

522 if len(top_contributors) > 1 and top_contributors[0][1] > 0.5:

523 return (

524 f"Drift primarily driven by {dim_name} "

525 f"({top_contributors[0][1]*100:.1f}% of total drift). "

526 f"Value changed from {vec_a[idx]:.4f} to {vec_b[idx]:.4f} "

527 f"({pct_diff:.1f}% difference)."

528 )

529 elif len(top_contributors) > 1:

530 top_names = [str(c[0]) for c, _ in zip(top_contributors[:3], range(3), strict=False)]

531 return (

532 f"Drift distributed across multiple dimensions. "

533 f"Top contributors: {', '.join(top_names)}. "

534 f"Largest single change in {dim_name}."

535 )

536 else:

537 return f"Single dimension drift in {dim_name}."

538

539

540 def _calculate_embedding_confidence(

541 vec_a: np.ndarray,

542 vec_b: np.ndarray,

543) -> float:

544 """Calculate confidence score for embedding verification."""

545 # Base confidence

546 confidence = 0.9

547

548 # Reduce confidence for very small vectors (less reliable)

549 if len(vec_a) < 10:

550 confidence *= 0.9

551

552 # Reduce confidence if vectors have very different magnitudes

553 norm_a = np.linalg.norm(vec_a)

554 norm_b = np.linalg.norm(vec_b)

555 if norm_a > 0 and norm_b > 0:

556 magnitude_ratio = min(norm_a, norm_b) / max(norm_a, norm_b)

557 if magnitude_ratio < 0.5:

558 confidence *= 0.85

559

560 # Reduce confidence for near-zero vectors

561 if norm_a < 1e-6 or norm_b < 1e-6:

562 confidence *= 0.7

563

564 return float(np.clip(confidence, 0.0, 1.0))

565

566

567 def _log_to_audit(

568 audit_trail: AuditTrail,

569 vec_a: np.ndarray,

570 vec_b: np.ndarray,

571 result: VerificationScore,

572 metric: str,

573 profile_name: str | None,

574) -> None:

575 """Log verification to audit trail."""

576 passed = result.details.get("profile", {}).get("passed", result.drift_score <= 0.3)

577

578 audit_trail.log(

579 operation="verify_embeddings",

580 inputs={

581 "embedding_a_shape": vec_a.shape,

582 "embedding_b_shape": vec_b.shape,

583 "embedding_a_norm": float(np.linalg.norm(vec_a)),

584 "embedding_b_norm": float(np.linalg.norm(vec_b)),

585 },

586 drift_score=result.drift_score,

587 confidence=result.confidence,

588 metric_used=metric,

589 profile_used=profile_name,

590 passed=passed,

591 result_details={

592 "drift_type": result.drift_type.value,

593 "raw_distance": result.details.get("raw_distance"),

594 },

595 )

596

597

598 def verify_distributions(dist_a: ArrayLike, dist_b: ArrayLike) -> VerificationScore:

599 """

600 Calculate drift between two probability distributions.

601

602 Uses KL divergence and other statistical measures to compare distributions.

603

604 Args:

605 dist_a: First probability distribution

606 dist_b: Second probability distribution

607

608 Returns:

609 VerificationScore with distribution-based drift score

610 """

611 p = np.asarray(dist_a, dtype=np.float64)

612 q = np.asarray(dist_b, dtype=np.float64)

613

614 # Normalize to valid probability distributions

615 p = p / (p.sum() + 1e-10)

616 q = q / (q.sum() + 1e-10)

617

618 # Add small epsilon to avoid log(0)

619 eps = 1e-10

620 p = np.clip(p, eps, 1.0)

621 q = np.clip(q, eps, 1.0)

622

623 if HAS_SCIPY:

624 # KL divergence

625 kl_div = stats.entropy(p, q)

626 # Jensen-Shannon divergence (symmetric, bounded [0, 1])

627 m = 0.5 * (p + q)

628 js_div = 0.5 * stats.entropy(p, m) + 0.5 * stats.entropy(q, m)

629 else:

630 # Fallback implementations

631 kl_div = float(np.sum(p * np.log(p / q)))

632 m = 0.5 * (p + q)

633 js_div = 0.5 * np.sum(p * np.log(p / m)) + 0.5 * np.sum(q * np.log(q / m))

634

635 # Total variation distance

636 tv_dist = 0.5 * np.sum(np.abs(p - q))

637

638 # Combined drift (JS divergence is bounded [0, ln(2)])

639 drift_score = js_div / np.log(2) # Normalize to [0, 1]

640

641 return VerificationScore(

642 drift_score=float(np.clip(drift_score, 0.0, 1.0)),

643 confidence=0.9,

644 drift_type=DriftType.NUMERICAL,

645 details={

646 "kl_divergence": float(kl_div),

647 "js_divergence": float(js_div),

648 "total_variation": float(tv_dist),

649 },

650 )

651

652

653 def verify_sequences(seq_a: Sequence[str], seq_b: Sequence[str]) -> VerificationScore:

654 """

655 Calculate drift between two sequences of tokens/items.

656

657 Uses edit distance and sequence alignment metrics.

658

659 Args:

660 seq_a: First sequence

661 seq_b: Second sequence

662

663 Returns:

664 VerificationScore with sequence-based drift score

665 """

666 if not seq_a and not seq_b:

667 return VerificationScore(

668 drift_score=0.0,

669 confidence=1.0,

670 drift_type=DriftType.LEXICAL,

671 details={"reason": "both_empty"},

672 )

673

674 # Levenshtein distance

675 edit_dist = _levenshtein_distance(seq_a, seq_b)

676 max_len = max(len(seq_a), len(seq_b))

677 normalized_edit = edit_dist / max_len if max_len > 0 else 0.0

678

679 # Jaccard similarity (set-based)

680 set_a = set(seq_a)

681 set_b = set(seq_b)

682 intersection = len(set_a & set_b)

683 union = len(set_a | set_b)

684 jaccard = intersection / union if union > 0 else 1.0

685 jaccard_drift = 1.0 - jaccard

686

687 # Order-aware similarity (longest common subsequence)

688 lcs_len = _lcs_length(seq_a, seq_b)

689 lcs_ratio = 2 * lcs_len / (len(seq_a) + len(seq_b)) if (len(seq_a) + len(seq_b)) > 0 else 1.0

690 lcs_drift = 1.0 - lcs_ratio

691

692 # Combined

693 drift_score = 0.4 * normalized_edit + 0.3 * jaccard_drift + 0.3 * lcs_drift

694

695 return VerificationScore(

696 drift_score=float(np.clip(drift_score, 0.0, 1.0)),

697 confidence=0.85,

698 drift_type=DriftType.STRUCTURAL,

699 details={

700 "edit_distance": edit_dist,

701 "normalized_edit": float(normalized_edit),

702 "jaccard_similarity": float(jaccard),

703 "lcs_ratio": float(lcs_ratio),

704 },

705 )

706

707

708# ============================================================================

709# Internal pure functions

710# ============================================================================

711

712

713 def _lexical_drift(text_a: str, text_b: str) -> dict:

714 """

715 Calculate lexical drift between two texts.

716

717 Pure function - no side effects.

718 """

719 # Character-level comparison

720 chars_a = set(text_a)

721 chars_b = set(text_b)

722 char_jaccard = len(chars_a & chars_b) / len(chars_a | chars_b) if (chars_a | chars_b) else 1.0

723

724 # Word-level comparison

725 words_a = set(text_a.split())

726 words_b = set(text_b.split())

727 word_jaccard = len(words_a & words_b) / len(words_a | words_b) if (words_a | words_b) else 1.0

728

729 # Length ratio

730 len_a, len_b = len(text_a), len(text_b)

731 length_ratio = min(len_a, len_b) / max(len_a, len_b) if max(len_a, len_b) > 0 else 1.0

732

733 # Combined score (lower similarity = higher drift)

734 similarity = 0.3 * char_jaccard + 0.5 * word_jaccard + 0.2 * length_ratio

735 drift = 1.0 - similarity

736

737 return {

738 "score": drift,

739 "char_jaccard": char_jaccard,

740 "word_jaccard": word_jaccard,

741 "length_ratio": length_ratio,

742 }

743

744

745 def _structural_drift(text_a: str, text_b: str) -> dict:

746 """

747 Calculate structural drift between two texts.

748

749 Analyzes structure like line count, indentation, code patterns.

750 Pure function - no side effects.

751 """

752 lines_a = text_a.split("\n")

753 lines_b = text_b.split("\n")

754

755 # Line count difference

756 line_count_a, line_count_b = len(lines_a), len(lines_b)

757 line_ratio = (

758 min(line_count_a, line_count_b) / max(line_count_a, line_count_b)

759 if max(line_count_a, line_count_b) > 0

760 else 1.0

761 )

762

763 # Indentation pattern

764 indent_a = [len(line) - len(line.lstrip()) for line in lines_a if line.strip()]

765 indent_b = [len(line) - len(line.lstrip()) for line in lines_b if line.strip()]

766

767 if indent_a and indent_b:

768 avg_indent_a = np.mean(indent_a)

769 avg_indent_b = np.mean(indent_b)

770 max_indent = max(avg_indent_a, avg_indent_b, 1)

771 indent_similarity = 1.0 - abs(avg_indent_a - avg_indent_b) / max_indent

772 else:

773 indent_similarity = 1.0 if (not indent_a and not indent_b) else 0.5

774

775 # Code pattern markers (for code comparison)

776 patterns = ["def ", "class ", "import ", "return ", "if ", "for ", "while ", "try:", "except"]

777 pattern_a = {p for p in patterns if p in text_a}

778 pattern_b = {p for p in patterns if p in text_b}

779 pattern_jaccard = (

780 len(pattern_a & pattern_b) / len(pattern_a | pattern_b) if (pattern_a | pattern_b) else 1.0

781 )

782

783 # Combined

784 similarity = 0.3 * line_ratio + 0.3 * indent_similarity + 0.4 * pattern_jaccard

785 drift = 1.0 - similarity

786

787 return {

788 "score": drift,

789 "line_ratio": line_ratio,

790 "indent_similarity": indent_similarity,

791 "pattern_jaccard": pattern_jaccard,

792 }

793

794

795 def _numerical_drift(text_a: str, text_b: str) -> dict:

796 """

797 Calculate numerical drift by extracting and comparing numbers.

798

799 Pure function - no side effects.

800 """

801 import re

802

803 # Extract numbers from both texts

804 number_pattern = r"-?\d+\.?\d*"

805 numbers_a = [float(n) for n in re.findall(number_pattern, text_a)]

806 numbers_b = [float(n) for n in re.findall(number_pattern, text_b)]

807

808 if not numbers_a and not numbers_b:

809 return {"score": 0.0, "reason": "no_numbers"}

810

811 if not numbers_a or not numbers_b:

812 return {"score": 0.5, "reason": "numbers_only_in_one"}

813

814 # Compare statistics

815 mean_a, mean_b = np.mean(numbers_a), np.mean(numbers_b)

816 std_a, std_b = np.std(numbers_a), np.std(numbers_b)

817

818 # Relative difference in means

819 max_mean = max(abs(mean_a), abs(mean_b), 1e-10)

820 mean_diff = abs(mean_a - mean_b) / max_mean

821

822 # Relative difference in stds

823 max_std = max(std_a, std_b, 1e-10)

824 std_diff = abs(std_a - std_b) / max_std if max_std > 1e-10 else 0.0

825

826 # Count difference

827 count_ratio = min(len(numbers_a), len(numbers_b)) / max(len(numbers_a), len(numbers_b))

828

829 # Combined

830 drift = 0.4 * min(mean_diff, 1.0) + 0.3 * min(std_diff, 1.0) + 0.3 * (1.0 - count_ratio)

831

832 return {

833 "score": drift,

834 "mean_a": mean_a,

835 "mean_b": mean_b,

836 "std_a": std_a,

837 "std_b": std_b,

838 "count_a": len(numbers_a),

839 "count_b": len(numbers_b),

840 }

841

842

843 def _levenshtein_distance(seq_a: Sequence, seq_b: Sequence) -> int:

844 """

845 Calculate Levenshtein edit distance between two sequences.

846

847 Pure function using dynamic programming.

848 """

849 m, n = len(seq_a), len(seq_b)

850

851 if m == 0:

852 return n

853 if n == 0:

854 return m

855

856 # Use numpy for efficiency

857 dp = np.zeros((m + 1, n + 1), dtype=np.int32)

858 dp[:, 0] = np.arange(m + 1)

859 dp[0, :] = np.arange(n + 1)

860

861 for i in range(1, m + 1):

862 for j in range(1, n + 1):

863 cost = 0 if seq_a[i - 1] == seq_b[j - 1] else 1

864 dp[i, j] = min(

865 dp[i - 1, j] + 1, # deletion

866 dp[i, j - 1] + 1, # insertion

867 dp[i - 1, j - 1] + cost, # substitution

868 )

869

870 return int(dp[m, n])

871

872

873 def _lcs_length(seq_a: Sequence, seq_b: Sequence) -> int:

874 """

875 Calculate length of Longest Common Subsequence.

876

877 Pure function using dynamic programming.

878 """

879 m, n = len(seq_a), len(seq_b)

880

881 if m == 0 or n == 0:

882 return 0

883

884 dp = np.zeros((m + 1, n + 1), dtype=np.int32)

885

886 for i in range(1, m + 1):

887 for j in range(1, n + 1):

888 if seq_a[i - 1] == seq_b[j - 1]:

889 dp[i, j] = dp[i - 1, j - 1] + 1

890 else:

891 dp[i, j] = max(dp[i - 1, j], dp[i, j - 1])

892

893 return int(dp[m, n])

894

895

896# ============================================================================

897# Batch verification functions

898# ============================================================================

899

900

901 def verify_batch(outputs_a: Sequence[str], outputs_b: Sequence[str]) -> list[VerificationScore]:

902 """

903 Verify multiple output pairs.

904

905 Pure function that processes pairs in sequence.

906

907 Args:

908 outputs_a: Sequence of outputs from source A

909 outputs_b: Sequence of outputs from source B (same length as outputs_a)

910

911 Returns:

912 List of VerificationScore for each pair

913 """

914 if len(outputs_a) != len(outputs_b):

915 raise ValueError(

916 f"Length mismatch: outputs_a has {len(outputs_a)} items, "

917 f"outputs_b has {len(outputs_b)} items"

918 )

919

920 return [verify(a, b) for a, b in zip(outputs_a, outputs_b, strict=False)]

921

922

923 def aggregate_scores(scores: Sequence[VerificationScore]) -> dict:

924 """

925 Aggregate multiple verification scores into summary statistics.

926

927 Pure function.

928

929 Args:

930 scores: Sequence of VerificationScore objects

931

932 Returns:

933 Dictionary with aggregate statistics

934 """

935 if not scores:

936 return {"count": 0}

937

938 drift_values = [s.drift_score for s in scores]

939 confidence_values = [s.confidence for s in scores]

940

941 drift_types: dict[str, int] = {}

942 for s in scores:

943 drift_types[s.drift_type.value] = drift_types.get(s.drift_type.value, 0) + 1

944

945 return {

946 "count": len(scores),

947 "mean_drift": float(np.mean(drift_values)),

948 "std_drift": float(np.std(drift_values)),

949 "min_drift": float(np.min(drift_values)),

950 "max_drift": float(np.max(drift_values)),

951 "median_drift": float(np.median(drift_values)),

952 "mean_confidence": float(np.mean(confidence_values)),

953 "drift_type_distribution": drift_types,

954 }