fix(plugin-eval): surface plugin-level depth downgrades loudly (#532)

`plugin-eval certify <plugin-dir>` advertises a deep, three-layer
evaluation (static + judge + Monte Carlo) but `EvalEngine.evaluate_plugin`
only runs the static layer regardless of requested depth, since judge
and Monte Carlo are per-skill primitives. The docstring on that method
acknowledges this, but nothing surfaces it to the user:

  - The CLI emits no warning.
  - The markdown report prints `**Depth:** deep` even though only the
    static layer ran.
  - The user's only signal is a footnote ("No model usage") near the
    bottom of the report and the `Confidence: Estimated` row — both
    easy to miss when the requested depth said otherwise.

This change makes the downgrade impossible to miss without changing
the underlying eval behaviour (per-skill judge aggregation is a larger
feature, not a bug fix):

1. **CLI warning to stderr.** `_run_score` detects plugin-target runs
   at non-quick depth and prints a yellow `warning:` line to stderr
   naming the skipped layers and the workaround (run on a single skill
   to get the deeper layers).

2. **In-band markdown callout.** `Reporter` now derives the *effective*
   depth from the set of layers actually present in the result. When it
   differs from the requested depth, the report header reads
   `Depth: deep (requested) → quick (effective)` and a `> Note:` block
   above the score table explains why and how to get the deeper layers.

3. **Effective-depth helper.** `_effective_depth(result)` maps the set
   of layer names (`static` / `judge` / `monte_carlo`) back to a `Depth`
   value, so the reporter never has to trust `result.config.depth` when
   describing what actually ran.

Tests:
  - `TestDepthDowngradeWarning` (4 tests): asserts the helper, the
    "no warning when honored" path, and the warning content for both
    deep and standard requests.
  - `TestCLI` (2 new tests): asserts the stderr warning is emitted on
    plugin-level certify and is *not* emitted at quick depth.

Full plugin-eval suite (75 tests) passes.

plugins/plugin-eval/src/plugin_eval/cli.py

@@ -17,6 +17,7 @@ app = typer.Typer(
     add_completion=False,
 )
 console = Console()
+stderr_console = Console(stderr=True)
 
 
 def _detect_target(path: Path) -> str:
@@ -55,6 +56,15 @@ def _run_score(
     if target == "skill":
         result = engine.evaluate_skill(path)
     elif target == "plugin":
+        if depth != Depth.QUICK:
+            stderr_console.print(
+                f"[yellow]warning:[/yellow] plugin-level evaluation only runs the "
+                f"static layer; judge and Monte Carlo layers require per-skill "
+                f"evaluation. Requested depth [bold]{depth.value}[/bold] will be "
+                f"served from the static layer only — confidence label will be "
+                f"[bold]Estimated[/bold] regardless. To use the deeper layers, "
+                f"point at an individual skill directory."
+            )
         result = engine.evaluate_plugin(path)
     else:
         # Attempt skill evaluation as fallback

plugins/plugin-eval/src/plugin_eval/reporter.py

@@ -2,7 +2,20 @@
 
 from __future__ import annotations
 
-from plugin_eval.models import PluginEvalResult
+from plugin_eval.models import Depth, PluginEvalResult
+
+
+_LAYER_TO_DEPTH: dict[frozenset[str], Depth] = {
+    frozenset({"static", "judge", "monte_carlo"}): Depth.DEEP,
+    frozenset({"static", "judge"}): Depth.STANDARD,
+    frozenset({"static"}): Depth.QUICK,
+}
+
+
+def _effective_depth(result: PluginEvalResult) -> Depth:
+    """Return the deepest depth actually covered by the layers that ran."""
+    layer_names = frozenset(layer.layer for layer in result.layers)
+    return _LAYER_TO_DEPTH.get(layer_names, Depth.QUICK)
 
 
 class Reporter:
@@ -27,9 +40,29 @@ class Reporter:
         lines.append("")
         lines.append(f"**Path:** `{result.plugin_path}`")
         lines.append(f"**Timestamp:** {result.timestamp}")
-        lines.append(f"**Depth:** {result.config.depth}")
+        requested = Depth(result.config.depth)
+        effective = _effective_depth(result)
+        if effective is requested:
+            lines.append(f"**Depth:** {requested.value}")
+        else:
+            lines.append(
+                f"**Depth:** {requested.value} (requested) → "
+                f"{effective.value} (effective)"
+            )
         lines.append("")
 
+        if effective is not requested:
+            lines.append(
+                "> **Note:** Requested depth `"
+                f"{requested.value}` was downgraded to `{effective.value}` "
+                "because plugin-level evaluation only runs the static layer. "
+                "Judge and Monte Carlo layers require per-skill evaluation — "
+                "point at an individual skill directory to use the deeper "
+                "layers. Composite score and confidence reflect the layers "
+                "actually run."
+            )
+            lines.append("")
+
         # Overall Score
         lines.append("## Overall Score")
         lines.append("")

plugins/plugin-eval/tests/test_cli.py

@@ -29,3 +29,31 @@ class TestCLI:
     def test_score_nonexistent_path(self, tmp_path: Path):
         result = runner.invoke(app, ["score", str(tmp_path / "nonexistent")])
         assert result.exit_code == 2
+
+    def test_plugin_eval_at_deep_depth_emits_downgrade_warning(
+        self, sample_plugin_dir: Path
+    ) -> None:
+        """Plugin-level evaluation only runs the static layer; certify-style
+        invocations at deep depth must warn the user that the deeper layers
+        were skipped, not silently produce a static-only report.
+        """
+        result = runner.invoke(
+            app,
+            ["certify", str(sample_plugin_dir), "--output", "markdown"],
+        )
+        assert result.exit_code == 0
+        # Click 8.3+ exposes stdout/stderr as separate attributes by default.
+        assert "warning" in result.stderr.lower()
+        assert "plugin-level" in result.stderr.lower()
+        assert "deep" in result.stderr.lower()
+
+    def test_plugin_eval_at_quick_depth_does_not_warn(
+        self, sample_plugin_dir: Path
+    ) -> None:
+        """No warning when the requested depth is already static-only."""
+        result = runner.invoke(
+            app,
+            ["score", str(sample_plugin_dir), "--depth", "quick"],
+        )
+        assert result.exit_code == 0
+        assert "warning" not in result.stderr.lower()

plugins/plugin-eval/tests/test_reporter.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 from plugin_eval.engine import EvalEngine
 from plugin_eval.models import Depth, EvalConfig
-from plugin_eval.reporter import Reporter
+from plugin_eval.reporter import Reporter, _effective_depth
 
 
 class TestReporter:
@@ -30,3 +30,52 @@ class TestReporter:
         assert "Overall Score" in output
         assert "Layer Breakdown" in output
         assert "Dimension Scores" in output
+
+
+class TestDepthDowngradeWarning:
+    """When plugin-level evaluation silently downgrades a deep/standard request
+    to static-only, the reporter must surface the downgrade in-band so the
+    consumer cannot mistake the score for a deeply-evaluated one.
+    """
+
+    def test_effective_depth_matches_layers_run(self, sample_skill_dir: Path) -> None:
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+        assert _effective_depth(result) is Depth.QUICK
+
+    def test_markdown_shows_no_warning_when_depth_was_honored(
+        self, sample_skill_dir: Path
+    ) -> None:
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+
+        output = Reporter().to_markdown(result)
+        assert "(requested)" not in output
+        assert "downgraded" not in output
+
+    def test_markdown_shows_warning_when_plugin_eval_downgrades_depth(
+        self, sample_plugin_dir: Path
+    ) -> None:
+        # Plugin-level eval at deep depth: the engine runs only the static
+        # layer regardless. The report must say so clearly.
+        config = EvalConfig(depth=Depth.DEEP)
+        engine = EvalEngine(config)
+        result = engine.evaluate_plugin(sample_plugin_dir)
+
+        output = Reporter().to_markdown(result)
+        assert "deep (requested)" in output
+        assert "quick (effective)" in output
+        assert "downgraded" in output
+
+    def test_markdown_shows_warning_when_standard_depth_is_downgraded(
+        self, sample_plugin_dir: Path
+    ) -> None:
+        config = EvalConfig(depth=Depth.STANDARD)
+        engine = EvalEngine(config)
+        result = engine.evaluate_plugin(sample_plugin_dir)
+
+        output = Reporter().to_markdown(result)
+        assert "standard (requested)" in output
+        assert "quick (effective)" in output