mirror of
https://github.com/wshobson/agents
synced 2026-06-21 14:13:58 +00:00
Seth Hobson
38cb914673
Adds MonteCarloAnalyzer with SimResult/MonteCarloConfig dataclasses, run_simulation async helper, and _compute_statistics using Wilson score CI, bootstrap CI, Clopper-Pearson CI, and coefficient of variation. Wires MC layer into evaluate_skill for Depth.DEEP and Depth.THOROUGH runs (50 and 100 runs respectively).
46 lines
1.9 KiB
Python
46 lines
1.9 KiB
Python
from pathlib import Path
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from plugin_eval.layers.monte_carlo import MonteCarloAnalyzer, MonteCarloConfig, SimResult
|
|
|
|
|
|
class TestSimResult:
|
|
def test_sim_result(self):
|
|
sr = SimResult(activated=True, quality_score=0.8, tokens=2500, duration_ms=1200)
|
|
assert sr.activated is True
|
|
assert sr.errored is False
|
|
|
|
|
|
class TestMonteCarloAnalyzer:
|
|
@pytest.mark.asyncio
|
|
@patch("plugin_eval.layers.monte_carlo.run_simulation")
|
|
async def test_run_with_mocked_sims(self, mock_sim, sample_skill_dir: Path):
|
|
mock_sim.return_value = SimResult(
|
|
activated=True, quality_score=0.82, tokens=2800, duration_ms=1500
|
|
)
|
|
config = MonteCarloConfig(n_runs=10, concurrency=2)
|
|
analyzer = MonteCarloAnalyzer(config)
|
|
result = await analyzer.analyze_skill(sample_skill_dir)
|
|
assert result.layer == "monte_carlo"
|
|
assert result.score > 0
|
|
assert "triggering" in result.sub_scores
|
|
assert "output_consistency" in result.sub_scores
|
|
assert "failure_rate" in result.sub_scores
|
|
|
|
def test_statistical_analysis(self):
|
|
"""Test the statistical analysis on pre-computed sim results."""
|
|
analyzer = MonteCarloAnalyzer(MonteCarloConfig(n_runs=50))
|
|
results = [
|
|
SimResult(activated=True, quality_score=0.8 + i * 0.002, tokens=2500, duration_ms=1200)
|
|
for i in range(48)
|
|
] + [
|
|
SimResult(activated=False, quality_score=0.0, tokens=500, duration_ms=200, errored=True),
|
|
SimResult(activated=True, quality_score=0.75, tokens=8000, duration_ms=5000),
|
|
]
|
|
stats = analyzer._compute_statistics(results)
|
|
assert stats["triggering"]["activation_rate"] == pytest.approx(0.98)
|
|
assert stats["failure_rate"]["p_fail"] == pytest.approx(0.02)
|
|
assert stats["output_consistency"]["cv"] < 0.15
|