feat(plugin-eval): implement Layer 2 LLM judge with Agent SDK and model tiering

plugins/plugin-eval/src/plugin_eval/engine.py

@@ -9,6 +9,7 @@ from plugin_eval.layers.static import StaticAnalyzer
 from plugin_eval.models import (
     Badge,
     CompositeResult,
+    Depth,
     DimensionScore,
     EvalConfig,
     LayerResult,
@@ -73,9 +74,20 @@ class EvalEngine:
         static_result = self._static.analyze_skill(skill_dir)
         layers.append(static_result)
 
-        # Layer 2: LLM Judge — will be implemented in Task 8
-        if "judge" in self.config.depth.layers:
-            pass  # placeholder
+        # Layer 2: Judge (standard+ depth)
+        if self.config.depth in (Depth.STANDARD, Depth.DEEP, Depth.THOROUGH):
+            import asyncio
+
+            from plugin_eval.layers.judge import JudgeAnalyzer, JudgeConfig
+
+            judge_config = JudgeConfig(
+                judges=self.config.judges,
+                auth=self.config.auth,
+                concurrency=self.config.concurrency,
+            )
+            judge = JudgeAnalyzer(judge_config)
+            judge_result = asyncio.run(judge.analyze_skill(skill_dir))
+            layers.append(judge_result)
 
         # Layer 3: Monte Carlo — will be implemented in Task 9
         if "monte_carlo" in self.config.depth.layers:

plugins/plugin-eval/src/plugin_eval/layers/judge.py

@@ -0,0 +1,277 @@
+"""Layer 2: LLM Judge — semantic evaluation via Claude, model-tiered, async."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from plugin_eval.models import LayerResult
+from plugin_eval.parser import parse_skill
+
+# ---------------------------------------------------------------------------
+# Anchored rubrics
+# ---------------------------------------------------------------------------
+
+ORCHESTRATION_RUBRIC = """
+Score 0.0 — Poor: Skill acts as standalone agent; manages its own tool calls and sub-tasks.
+Score 0.25 — Below average: Skill has some orchestration logic mixed with worker tasks.
+Score 0.5 — Average: Skill delegates some tasks but still coordinates multi-step flows itself.
+Score 0.75 — Good: Skill is mostly a worker; inputs/outputs documented, minimal coordination.
+Score 1.0 — Excellent: Pure worker role; composable, clear contracts, no orchestration logic.
+""".strip()
+
+SCOPE_RUBRIC = """
+Score 0.0 — Too thin: Stub or trivial wrapper with near-zero unique value.
+Score 0.25 — Under-scoped: Covers only a narrow slice; misses obvious related tasks.
+Score 0.5 — Average: Reasonable scope but either too broad or somewhat narrow.
+Score 0.75 — Well-scoped: Covers one coherent domain; neither bloated nor sparse.
+Score 1.0 — Perfectly calibrated: Minimal surface area, maximum cohesion, ideal composability.
+""".strip()
+
+# ---------------------------------------------------------------------------
+# Model resolution
+# ---------------------------------------------------------------------------
+
+_MODEL_MAP: dict[str, str] = {
+    "haiku": "claude-haiku-4-5-20251001",
+    "sonnet": "claude-sonnet-4-6",
+    "opus": "claude-opus-4-6",
+}
+
+
+def _resolve_model(tier: str) -> str:
+    """Map a tier name to a full model ID."""
+    return _MODEL_MAP.get(tier, _MODEL_MAP["sonnet"])
+
+
+# ---------------------------------------------------------------------------
+# LLM query helper (abstracted for testability)
+# ---------------------------------------------------------------------------
+
+
+async def query_llm(prompt: str, system: str = "", model: str = "claude-sonnet-4-6") -> dict:
+    """Call Claude via the Agent SDK and return a parsed JSON dict.
+
+    Raises RuntimeError if claude-agent-sdk is not installed.
+    """
+    try:
+        import claude_agent_sdk as sdk  # type: ignore[import-untyped]
+    except ImportError as exc:
+        raise RuntimeError(
+            "claude-agent-sdk is required for LLM judge. "
+            "Install with: pip install plugin-eval[llm]"
+        ) from exc
+
+    full_prompt = prompt
+    if system:
+        full_prompt = f"{system}\n\n{prompt}"
+
+    result_text = ""
+    async for event in sdk.stream(full_prompt, model=model):
+        if hasattr(event, "text"):
+            result_text += event.text
+
+    # Try to parse JSON — handles raw JSON or JSON inside a markdown code block
+    # Strip markdown fences if present
+    stripped = result_text.strip()
+    fence_match = re.search(r"```(?:json)?\s*([\s\S]+?)\s*```", stripped)
+    if fence_match:
+        stripped = fence_match.group(1).strip()
+
+    return json.loads(stripped)
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class JudgeConfig:
+    judges: int = 1
+    auth: str = "max"
+    concurrency: int = 4
+    model_tier: str = "auto"
+
+
+# ---------------------------------------------------------------------------
+# Analyzer
+# ---------------------------------------------------------------------------
+
+
+class JudgeAnalyzer:
+    """Semantic skill evaluation using Claude as a judge."""
+
+    def __init__(self, config: JudgeConfig) -> None:
+        self.config = config
+        self._sem = asyncio.Semaphore(config.concurrency)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def analyze_skill(self, skill_dir: Path) -> LayerResult:
+        """Run all 4 assessments concurrently and return a LayerResult."""
+        triggering, orchestration, output_quality, scope = await asyncio.gather(
+            self.assess_triggering(skill_dir),
+            self.assess_orchestration(skill_dir),
+            self.assess_output_quality(skill_dir),
+            self.assess_scope(skill_dir),
+        )
+
+        # Weighted composite: triggering 0.30, orchestration 0.30, output 0.25, scope 0.15
+        score = (
+            triggering.get("f1", 0.5) * 0.30
+            + orchestration.get("score", 0.5) * 0.30
+            + output_quality.get("score", 0.5) * 0.25
+            + scope.get("score", 0.5) * 0.15
+        )
+        score = max(0.0, min(1.0, score))
+
+        sub_scores: dict[str, float] = {
+            "triggering_accuracy": triggering.get("f1", 0.5),
+            "orchestration_fitness": orchestration.get("score", 0.5),
+            "output_quality": output_quality.get("score", 0.5),
+            "scope_calibration": scope.get("score", 0.5),
+        }
+
+        metadata: dict = {
+            "triggering": triggering,
+            "orchestration": orchestration,
+            "output_quality": output_quality,
+            "scope": scope,
+        }
+
+        return LayerResult(
+            layer="judge",
+            score=score,
+            sub_scores=sub_scores,
+            metadata=metadata,
+        )
+
+    # ------------------------------------------------------------------
+    # Individual assessments
+    # ------------------------------------------------------------------
+
+    async def assess_triggering(self, skill_dir: Path) -> dict:
+        """Generate 10 synthetic prompts and classify triggering accuracy via Haiku."""
+        skill = parse_skill(skill_dir)
+        model = _resolve_model("haiku")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Given this skill description:
+
+<description>
+{skill.description}
+</description>
+
+Generate 10 synthetic user prompts: 5 that SHOULD trigger this skill and 5 that should NOT.
+For each prompt, also predict whether a typical Claude model would trigger this skill.
+
+Return JSON matching this schema:
+{{
+  "predictions": [
+    {{"prompt": "...", "should_trigger": true, "would_trigger": true}},
+    ...
+  ],
+  "precision": <float 0-1>,
+  "recall": <float 0-1>,
+  "f1": <float 0-1>
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_orchestration(self, skill_dir: Path) -> dict:
+        """Rate orchestration fitness using an anchored rubric via Sonnet."""
+        skill = parse_skill(skill_dir)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Evaluate this skill's orchestration fitness.
+
+A skill should be a pure WORKER — it should NOT orchestrate other tools or agents.
+It should accept clear inputs and produce clear outputs.
+
+Rubric:
+{ORCHESTRATION_RUBRIC}
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0 matching rubric>,
+  "reasoning": "<one sentence>",
+  "evidence": ["<quote or observation>", ...]
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_output_quality(self, skill_dir: Path) -> dict:
+        """Simulate 3 tasks and judge output quality via Sonnet."""
+        skill = parse_skill(skill_dir)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Simulate 3 realistic tasks this skill would handle, then evaluate the
+expected output quality based on the skill's instructions.
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0>,
+  "simulations": [
+    {{"task": "...", "expected_output": "...", "quality_notes": "..."}}
+  ]
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_scope(self, skill_dir: Path) -> dict:
+        """Evaluate scope calibration using an anchored rubric via Sonnet."""
+        skill = parse_skill(skill_dir)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Evaluate this skill's scope calibration.
+
+Rubric:
+{SCOPE_RUBRIC}
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0 matching rubric>,
+  "assessment": "<one sentence>"
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)

plugins/plugin-eval/tests/test_judge.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from plugin_eval.layers.judge import JudgeAnalyzer, JudgeConfig
+
+
+class TestJudgeConfig:
+    def test_default_config(self):
+        config = JudgeConfig()
+        assert config.judges == 1
+        assert config.auth == "max"
+
+
+class TestJudgeAnalyzer:
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_assess_triggering(self, mock_query, sample_skill_dir: Path):
+        mock_query.return_value = {
+            "predictions": [
+                {"prompt": "test logging", "should_trigger": True, "would_trigger": True},
+                {"prompt": "make coffee", "should_trigger": False, "would_trigger": False},
+            ],
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1": 1.0,
+        }
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.assess_triggering(sample_skill_dir)
+        assert result["f1"] == 1.0
+        mock_query.assert_called()
+
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_assess_orchestration(self, mock_query, sample_skill_dir: Path):
+        mock_query.return_value = {
+            "score": 0.82,
+            "reasoning": "Clean worker role with structured outputs.",
+            "evidence": ["Output format documented", "No orchestration logic"],
+        }
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.assess_orchestration(sample_skill_dir)
+        assert result["score"] == 0.82
+
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_full_analysis(self, mock_query, sample_skill_dir: Path):
+        mock_query.side_effect = [
+            {"f1": 0.85, "precision": 0.90, "recall": 0.80, "predictions": []},
+            {"score": 0.82, "reasoning": "Good", "evidence": []},
+            {"score": 0.79, "simulations": []},
+            {"score": 0.88, "assessment": "well-scoped"},
+        ]
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.analyze_skill(sample_skill_dir)
+        assert result.layer == "judge"
+        assert result.score > 0