mirror of
https://github.com/wshobson/agents
synced 2026-06-21 14:13:58 +00:00
jon
83d70bcc58
`plugin-eval certify <plugin-dir>` advertises a deep, three-layer
evaluation (static + judge + Monte Carlo) but `EvalEngine.evaluate_plugin`
only runs the static layer regardless of requested depth, since judge
and Monte Carlo are per-skill primitives. The docstring on that method
acknowledges this, but nothing surfaces it to the user:
- The CLI emits no warning.
- The markdown report prints `**Depth:** deep` even though only the
static layer ran.
- The user's only signal is a footnote ("No model usage") near the
bottom of the report and the `Confidence: Estimated` row — both
easy to miss when the requested depth said otherwise.
This change makes the downgrade impossible to miss without changing
the underlying eval behaviour (per-skill judge aggregation is a larger
feature, not a bug fix):
1. **CLI warning to stderr.** `_run_score` detects plugin-target runs
at non-quick depth and prints a yellow `warning:` line to stderr
naming the skipped layers and the workaround (run on a single skill
to get the deeper layers).
2. **In-band markdown callout.** `Reporter` now derives the *effective*
depth from the set of layers actually present in the result. When it
differs from the requested depth, the report header reads
`Depth: deep (requested) → quick (effective)` and a `> Note:` block
above the score table explains why and how to get the deeper layers.
3. **Effective-depth helper.** `_effective_depth(result)` maps the set
of layer names (`static` / `judge` / `monte_carlo`) back to a `Depth`
value, so the reporter never has to trust `result.config.depth` when
describing what actually ran.
Tests:
- `TestDepthDowngradeWarning` (4 tests): asserts the helper, the
"no warning when honored" path, and the warning content for both
deep and standard requests.
- `TestCLI` (2 new tests): asserts the stderr warning is emitted on
plugin-level certify and is *not* emitted at quick depth.
Full plugin-eval suite (75 tests) passes.
60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
from pathlib import Path
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
from plugin_eval.cli import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
class TestCLI:
|
|
def test_score_quick(self, sample_skill_dir: Path):
|
|
result = runner.invoke(app, ["score", str(sample_skill_dir), "--depth", "quick"])
|
|
assert result.exit_code == 0
|
|
|
|
def test_score_json_output(self, sample_skill_dir: Path):
|
|
result = runner.invoke(
|
|
app, ["score", str(sample_skill_dir), "--depth", "quick", "--output", "json"]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert '"composite"' in result.stdout
|
|
|
|
def test_score_markdown_output(self, sample_skill_dir: Path):
|
|
result = runner.invoke(
|
|
app, ["score", str(sample_skill_dir), "--depth", "quick", "--output", "markdown"]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "PluginEval Report" in result.stdout
|
|
|
|
def test_score_nonexistent_path(self, tmp_path: Path):
|
|
result = runner.invoke(app, ["score", str(tmp_path / "nonexistent")])
|
|
assert result.exit_code == 2
|
|
|
|
def test_plugin_eval_at_deep_depth_emits_downgrade_warning(
|
|
self, sample_plugin_dir: Path
|
|
) -> None:
|
|
"""Plugin-level evaluation only runs the static layer; certify-style
|
|
invocations at deep depth must warn the user that the deeper layers
|
|
were skipped, not silently produce a static-only report.
|
|
"""
|
|
result = runner.invoke(
|
|
app,
|
|
["certify", str(sample_plugin_dir), "--output", "markdown"],
|
|
)
|
|
assert result.exit_code == 0
|
|
# Click 8.3+ exposes stdout/stderr as separate attributes by default.
|
|
assert "warning" in result.stderr.lower()
|
|
assert "plugin-level" in result.stderr.lower()
|
|
assert "deep" in result.stderr.lower()
|
|
|
|
def test_plugin_eval_at_quick_depth_does_not_warn(
|
|
self, sample_plugin_dir: Path
|
|
) -> None:
|
|
"""No warning when the requested depth is already static-only."""
|
|
result = runner.invoke(
|
|
app,
|
|
["score", str(sample_plugin_dir), "--depth", "quick"],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "warning" not in result.stderr.lower()
|