mirror of
https://github.com/wshobson/agents
synced 2026-06-21 14:13:58 +00:00
jon
83d70bcc58
`plugin-eval certify <plugin-dir>` advertises a deep, three-layer
evaluation (static + judge + Monte Carlo) but `EvalEngine.evaluate_plugin`
only runs the static layer regardless of requested depth, since judge
and Monte Carlo are per-skill primitives. The docstring on that method
acknowledges this, but nothing surfaces it to the user:
- The CLI emits no warning.
- The markdown report prints `**Depth:** deep` even though only the
static layer ran.
- The user's only signal is a footnote ("No model usage") near the
bottom of the report and the `Confidence: Estimated` row — both
easy to miss when the requested depth said otherwise.
This change makes the downgrade impossible to miss without changing
the underlying eval behaviour (per-skill judge aggregation is a larger
feature, not a bug fix):
1. **CLI warning to stderr.** `_run_score` detects plugin-target runs
at non-quick depth and prints a yellow `warning:` line to stderr
naming the skipped layers and the workaround (run on a single skill
to get the deeper layers).
2. **In-band markdown callout.** `Reporter` now derives the *effective*
depth from the set of layers actually present in the result. When it
differs from the requested depth, the report header reads
`Depth: deep (requested) → quick (effective)` and a `> Note:` block
above the score table explains why and how to get the deeper layers.
3. **Effective-depth helper.** `_effective_depth(result)` maps the set
of layer names (`static` / `judge` / `monte_carlo`) back to a `Depth`
value, so the reporter never has to trust `result.config.depth` when
describing what actually ran.
Tests:
- `TestDepthDowngradeWarning` (4 tests): asserts the helper, the
"no warning when honored" path, and the warning content for both
deep and standard requests.
- `TestCLI` (2 new tests): asserts the stderr warning is emitted on
plugin-level certify and is *not* emitted at quick depth.
Full plugin-eval suite (75 tests) passes.
82 lines
3.0 KiB
Python
82 lines
3.0 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
from plugin_eval.engine import EvalEngine
|
|
from plugin_eval.models import Depth, EvalConfig
|
|
from plugin_eval.reporter import Reporter, _effective_depth
|
|
|
|
|
|
class TestReporter:
|
|
def test_json_output(self, sample_skill_dir: Path):
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
|
|
reporter = Reporter()
|
|
output = reporter.to_json(result)
|
|
parsed = json.loads(output)
|
|
assert "composite" in parsed
|
|
assert "layers" in parsed
|
|
assert parsed["composite"]["confidence_label"] == "Estimated"
|
|
|
|
def test_markdown_output(self, sample_skill_dir: Path):
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
|
|
reporter = Reporter()
|
|
output = reporter.to_markdown(result)
|
|
assert "# PluginEval Report" in output
|
|
assert "Overall Score" in output
|
|
assert "Layer Breakdown" in output
|
|
assert "Dimension Scores" in output
|
|
|
|
|
|
class TestDepthDowngradeWarning:
|
|
"""When plugin-level evaluation silently downgrades a deep/standard request
|
|
to static-only, the reporter must surface the downgrade in-band so the
|
|
consumer cannot mistake the score for a deeply-evaluated one.
|
|
"""
|
|
|
|
def test_effective_depth_matches_layers_run(self, sample_skill_dir: Path) -> None:
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
assert _effective_depth(result) is Depth.QUICK
|
|
|
|
def test_markdown_shows_no_warning_when_depth_was_honored(
|
|
self, sample_skill_dir: Path
|
|
) -> None:
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
|
|
output = Reporter().to_markdown(result)
|
|
assert "(requested)" not in output
|
|
assert "downgraded" not in output
|
|
|
|
def test_markdown_shows_warning_when_plugin_eval_downgrades_depth(
|
|
self, sample_plugin_dir: Path
|
|
) -> None:
|
|
# Plugin-level eval at deep depth: the engine runs only the static
|
|
# layer regardless. The report must say so clearly.
|
|
config = EvalConfig(depth=Depth.DEEP)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_plugin(sample_plugin_dir)
|
|
|
|
output = Reporter().to_markdown(result)
|
|
assert "deep (requested)" in output
|
|
assert "quick (effective)" in output
|
|
assert "downgraded" in output
|
|
|
|
def test_markdown_shows_warning_when_standard_depth_is_downgraded(
|
|
self, sample_plugin_dir: Path
|
|
) -> None:
|
|
config = EvalConfig(depth=Depth.STANDARD)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_plugin(sample_plugin_dir)
|
|
|
|
output = Reporter().to_markdown(result)
|
|
assert "standard (requested)" in output
|
|
assert "quick (effective)" in output
|