feat(plugin-eval): implement Elo ranking system and corpus management

plugins/plugin-eval/src/plugin_eval/corpus.py

@@ -0,0 +1,132 @@
+"""Gold standard corpus management for Elo ranking."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from plugin_eval.parser import parse_skill
+
+
+@dataclass
+class CorpusEntry:
+    name: str
+    path: str
+    category: str
+    line_count: int
+    elo_rating: float = 1500.0
+
+
+class Corpus:
+    def __init__(self, corpus_dir: Path) -> None:
+        self.corpus_dir = corpus_dir
+        self.entries: list[CorpusEntry] = []
+        self._load()
+
+    @classmethod
+    def init_from_source(cls, plugins_dir: Path, corpus_dir: Path) -> Corpus:
+        """Index all skills from a plugins directory into a corpus."""
+        corpus_dir.mkdir(parents=True, exist_ok=True)
+
+        entries = []
+        for plugin_dir in sorted(plugins_dir.iterdir()):
+            if not plugin_dir.is_dir():
+                continue
+            skills_dir = plugin_dir / "skills"
+            if not skills_dir.exists():
+                continue
+            for skill_dir in sorted(skills_dir.iterdir()):
+                if skill_dir.is_dir() and (skill_dir / "SKILL.md").exists():
+                    try:
+                        skill = parse_skill(skill_dir)
+                        entries.append(
+                            CorpusEntry(
+                                name=skill.name,
+                                path=str(skill_dir),
+                                category=plugin_dir.name,
+                                line_count=skill.line_count,
+                            )
+                        )
+                    except Exception:
+                        continue
+
+        index = [
+            {
+                "name": e.name,
+                "path": e.path,
+                "category": e.category,
+                "line_count": e.line_count,
+                "elo_rating": e.elo_rating,
+            }
+            for e in entries
+        ]
+        (corpus_dir / "index.json").write_text(json.dumps(index, indent=2))
+
+        corpus = cls(corpus_dir)
+        return corpus
+
+    @property
+    def size(self) -> int:
+        return len(self.entries)
+
+    def list_skills(self) -> list[CorpusEntry]:
+        return self.entries
+
+    def select_references(
+        self,
+        category: str | None = None,
+        line_count: int | None = None,
+        n: int = 5,
+    ) -> list[CorpusEntry]:
+        """Select reference skills for Elo comparison."""
+        candidates = self.entries
+
+        if category:
+            same_cat = [e for e in candidates if e.category == category]
+            if same_cat:
+                candidates = same_cat
+
+        if line_count:
+            margin = line_count * 0.3
+            sized = [e for e in candidates if abs(e.line_count - line_count) <= margin]
+            if sized:
+                candidates = sized
+
+        candidates = sorted(candidates, key=lambda e: abs(e.elo_rating - 1500))
+        return candidates[:n]
+
+    def update_rating(self, name: str, new_rating: float) -> None:
+        for entry in self.entries:
+            if entry.name == name:
+                entry.elo_rating = new_rating
+                break
+        self._save()
+
+    def _load(self) -> None:
+        index_path = self.corpus_dir / "index.json"
+        if index_path.exists():
+            data = json.loads(index_path.read_text())
+            self.entries = [
+                CorpusEntry(
+                    name=e["name"],
+                    path=e["path"],
+                    category=e["category"],
+                    line_count=e["line_count"],
+                    elo_rating=e.get("elo_rating", 1500.0),
+                )
+                for e in data
+            ]
+
+    def _save(self) -> None:
+        index = [
+            {
+                "name": e.name,
+                "path": e.path,
+                "category": e.category,
+                "line_count": e.line_count,
+                "elo_rating": e.elo_rating,
+            }
+            for e in self.entries
+        ]
+        (self.corpus_dir / "index.json").write_text(json.dumps(index, indent=2))

plugins/plugin-eval/src/plugin_eval/elo.py

@@ -0,0 +1,50 @@
+"""Elo rating system for pairwise skill comparison."""
+
+from __future__ import annotations
+
+import random
+
+
+class EloCalculator:
+    def __init__(self, k_factor: int = 32) -> None:
+        self.k_factor = k_factor
+
+    def expected(self, rating_a: float, rating_b: float) -> float:
+        """Expected score for player A against player B."""
+        return 1.0 / (1.0 + 10 ** ((rating_b - rating_a) / 400))
+
+    def update(self, rating: float, opponent_rating: float, actual: float) -> float:
+        """Update rating after a single matchup."""
+        exp = self.expected(rating, opponent_rating)
+        return rating + self.k_factor * (actual - exp)
+
+    def compute_rating(self, initial: float, matchups: list[tuple[float, float]]) -> float:
+        """Compute final rating from (opponent_rating, actual_score) matchups."""
+        rating = initial
+        for opponent_rating, actual in matchups:
+            rating = self.update(rating, opponent_rating, actual)
+        return rating
+
+    def compute_rating_with_ci(
+        self,
+        initial: float,
+        matchups: list[tuple[float, float]],
+        n_resamples: int = 500,
+        seed: int | None = None,
+    ) -> tuple[float, float, float]:
+        """Compute rating with bootstrap CI by resampling matchups."""
+        point_estimate = self.compute_rating(initial, matchups)
+
+        if len(matchups) < 2:
+            return point_estimate, point_estimate, point_estimate
+
+        rng = random.Random(seed)
+        ratings = []
+        for _ in range(n_resamples):
+            sample = [rng.choice(matchups) for _ in range(len(matchups))]
+            ratings.append(self.compute_rating(initial, sample))
+
+        ratings.sort()
+        lower_idx = int(0.025 * n_resamples)
+        upper_idx = int(0.975 * n_resamples) - 1
+        return point_estimate, ratings[lower_idx], ratings[upper_idx]

plugins/plugin-eval/tests/test_corpus.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.corpus import Corpus
+
+
+class TestCorpus:
+    def test_init_from_plugins(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        assert corpus.size > 0
+        assert (corpus_dir / "index.json").exists()
+
+    def test_select_references(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        refs = corpus.select_references(category="development", n=3)
+        assert len(refs) <= 3
+
+    def test_list_skills(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        skills = corpus.list_skills()
+        assert len(skills) > 0

plugins/plugin-eval/tests/test_elo.py

@@ -0,0 +1,48 @@
+import pytest
+
+from plugin_eval.elo import EloCalculator
+
+
+class TestEloCalculator:
+    def test_expected_score(self):
+        calc = EloCalculator(k_factor=32)
+        assert calc.expected(1500, 1500) == pytest.approx(0.5)
+        assert calc.expected(1600, 1500) > 0.5
+        assert calc.expected(1400, 1500) < 0.5
+
+    def test_update_win(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=1.0)
+        assert new_rating > 1500
+        assert new_rating == pytest.approx(1516.0)
+
+    def test_update_loss(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=0.0)
+        assert new_rating < 1500
+
+    def test_update_draw(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=0.5)
+        assert new_rating == pytest.approx(1500.0)
+
+    def test_compute_rating_from_matchups(self):
+        calc = EloCalculator(k_factor=32)
+        matchups = [
+            (1540, 1.0),
+            (1500, 0.0),
+            (1460, 1.0),
+        ]
+        final = calc.compute_rating(1500, matchups)
+        assert final > 1500
+
+    def test_bootstrap_ci(self):
+        calc = EloCalculator(k_factor=32)
+        matchups = [
+            (1540, 1.0),
+            (1500, 0.0),
+            (1460, 1.0),
+            (1520, 0.5),
+        ]
+        rating, lower, upper = calc.compute_rating_with_ci(1500, matchups, seed=42)
+        assert lower <= rating <= upper or lower == upper