wshobson-agents/plugins/plugin-eval/tests/test_static.py

from pathlib import Path

import pytest

from plugin_eval.layers.static import _TRIGGER_PATTERN, StaticAnalyzer
from plugin_eval.models import LayerResult


def _make_skill(tmp_path: Path, description: str, name: str = "test-skill") -> Path:
    skill_dir = tmp_path / name
    skill_dir.mkdir()
    (skill_dir / "SKILL.md").write_text(
        f"---\nname: {name}\ndescription: \"{description}\"\n---\n\n"
        "# Skill\n\n## Overview\n\nBody.\n"
    )
    return skill_dir


class TestStaticAnalyzer:
    def test_analyze_valid_skill(self, sample_skill_dir: Path):
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(sample_skill_dir)
        assert isinstance(result, LayerResult)
        assert result.layer == "static"
        assert result.score > 0.5
        assert len(result.anti_patterns) == 0

    def test_analyze_poor_skill(self, poor_skill_dir: Path):
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(poor_skill_dir)
        assert result.score < 0.7
        flags = [ap.flag for ap in result.anti_patterns]
        assert "OVER_CONSTRAINED" in flags
        assert "MISSING_TRIGGER" in flags

    def test_analyze_plugin(self, sample_plugin_dir: Path):
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_plugin(sample_plugin_dir)
        assert result.layer == "static"
        assert result.score > 0.5
        assert "skill_scores" in result.sub_scores
        assert "agent_scores" in result.sub_scores

    def test_anti_pattern_penalty(self):
        analyzer = StaticAnalyzer()
        assert analyzer._anti_pattern_penalty(0) == 1.0
        assert analyzer._anti_pattern_penalty(2) == pytest.approx(0.9)
        assert analyzer._anti_pattern_penalty(10) == 0.5
        assert analyzer._anti_pattern_penalty(20) == 0.5

    def test_description_pushiness_score(self):
        analyzer = StaticAnalyzer()
        good = "Test skill for evaluation. Use when testing plugin-eval. Use PROACTIVELY for quality checks."
        weak = "A skill."
        assert analyzer._description_pushiness(good) > analyzer._description_pushiness(weak)


class TestTriggerPattern:
    """Regression coverage for the broadened trigger-phrase matcher.

    plugin-dev's canonical recommendation is third-person ("This skill should be
    used when …"), and several real-world plugins use prepositional triggers
    ("Use after …", "Use before …"). The pre-2026 regex only matched the
    imperative "Use when …" form, which produced false-positive MISSING_TRIGGER
    flags against Anthropic's own examples.
    """

    @pytest.mark.parametrize(
        "description",
        [
            "Use when testing plugin-eval.",
            "Use this skill when scaffolding plugins.",
            "This skill should be used when the user asks to 'create a hook'.",
            "Used when several attempts have failed in a row.",
            "Use after editing the source-of-truth files, before committing.",
            "Use before declaring a task complete after a hard debugging session.",
            "Use immediately before a commit, push, or edit-after-failure.",
            "Use whenever you are asked to plan inside a Paperclip company.",
            "Auto-loads when working on test files.",
            "Trigger when a Bash command fails three times in a row.",
            "Use PROACTIVELY before merging.",
        ],
    )
    def test_pattern_matches_canonical_forms(self, description: str) -> None:
        assert _TRIGGER_PATTERN.search(description), (
            f"Expected trigger phrase to match in: {description!r}"
        )

    @pytest.mark.parametrize(
        "description",
        [
            "A skill.",
            "Provides hook guidance.",
            "Returns the current timestamp.",
            "Performs static analysis on plugin directories.",
        ],
    )
    def test_pattern_rejects_descriptions_without_trigger(self, description: str) -> None:
        assert not _TRIGGER_PATTERN.search(description), (
            f"Did not expect trigger match in: {description!r}"
        )

    def test_third_person_skill_does_not_flag_missing_trigger(self, tmp_path: Path) -> None:
        """Anthropic plugin-dev's canonical phrasing must not be flagged."""
        skill_dir = _make_skill(
            tmp_path,
            "This skill should be used when the user asks to 'create a hook', "
            "'add a PreToolUse hook', or 'validate tool use'.",
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" not in flags

    def test_prepositional_trigger_does_not_flag_missing_trigger(self, tmp_path: Path) -> None:
        skill_dir = _make_skill(
            tmp_path,
            "Self-check before a single risky action. Use immediately before a "
            "commit, push, edit-after-failure, or skip-a-verification step.",
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" not in flags


def _make_skill_with_frontmatter(
    tmp_path: Path, frontmatter_lines: list[str], name: str = "test-skill"
) -> Path:
    skill_dir = tmp_path / name
    skill_dir.mkdir()
    frontmatter = "\n".join(frontmatter_lines)
    (skill_dir / "SKILL.md").write_text(
        f"---\n{frontmatter}\n---\n\n# Skill\n\n## Overview\n\nBody.\n"
    )
    return skill_dir


class TestTriggerExemptions:
    """`disable-model-invocation: true` and `paths:` frontmatter should exempt
    a skill from the MISSING_TRIGGER check, because those skills are not
    auto-invoked from the description.
    """

    def test_disable_model_invocation_exempts_skill(self, tmp_path: Path) -> None:
        skill_dir = _make_skill_with_frontmatter(
            tmp_path,
            [
                "name: setup",
                "description: One-time setup that adds .claude/state/ to the project's .gitignore.",
                "disable-model-invocation: true",
            ],
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" not in flags, (
            "Slash-only skills should not be flagged for missing description trigger"
        )

    def test_paths_auto_load_exempts_skill(self, tmp_path: Path) -> None:
        skill_dir = _make_skill_with_frontmatter(
            tmp_path,
            [
                "name: self-evaluate",
                "description: Self-critical evaluation guard for test/spec files.",
                'paths: "**/*test*,**/*spec*"',
            ],
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" not in flags, (
            "Path-triggered skills should not be flagged for missing description trigger"
        )

    def test_disable_model_invocation_false_still_checks_trigger(
        self, tmp_path: Path
    ) -> None:
        skill_dir = _make_skill_with_frontmatter(
            tmp_path,
            [
                "name: model-invocable",
                "description: A description without a trigger phrase whatsoever.",
                "disable-model-invocation: false",
            ],
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" in flags, (
            "Model-invocable skills without a trigger phrase must still be flagged"
        )

    def test_empty_paths_value_still_checks_trigger(self, tmp_path: Path) -> None:
        skill_dir = _make_skill_with_frontmatter(
            tmp_path,
            [
                "name: bad-paths",
                "description: Some skill description that lacks the trigger phrase.",
                'paths: ""',
            ],
        )
        analyzer = StaticAnalyzer()
        result = analyzer.analyze_skill(skill_dir)
        flags = [ap.flag for ap in result.anti_patterns]
        assert "MISSING_TRIGGER" in flags