feat(plugin-eval): add markdown/frontmatter parser for skills, agents, plugins

plugins/plugin-eval/src/plugin_eval/parser.py

@@ -0,0 +1,193 @@
+"""Parse Claude Code plugin structure: skills, agents, plugin.json."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+
+@dataclass
+class ParsedSkill:
+    path: Path
+    name: str
+    description: str
+    line_count: int
+    h2_count: int
+    h3_count: int
+    code_block_count: int
+    code_block_languages: list[str]
+    has_examples: bool
+    has_troubleshooting: bool
+    has_references: bool
+    has_assets: bool
+    reference_files: list[str]
+    asset_files: list[str]
+    total_content_lines: int
+    must_never_always_count: int
+    cross_references: list[str]
+    raw_content: str
+    frontmatter: dict
+
+
+@dataclass
+class ParsedAgent:
+    path: Path
+    name: str
+    description: str
+    model: str | None
+    has_tools_restriction: bool
+    tools: list[str]
+    has_proactive_trigger: bool
+    skill_references: list[str]
+    raw_content: str
+    frontmatter: dict
+
+
+@dataclass
+class ParsedPlugin:
+    path: Path
+    name: str
+    skills: list[ParsedSkill] = field(default_factory=list)
+    agents: list[ParsedAgent] = field(default_factory=list)
+    plugin_json: dict = field(default_factory=dict)
+
+
+def parse_skill(skill_dir: Path) -> ParsedSkill:
+    """Parse a skill directory into structured data."""
+    skill_md = skill_dir / "SKILL.md"
+    if not skill_md.exists():
+        raise FileNotFoundError(f"No SKILL.md found in {skill_dir}")
+
+    content = skill_md.read_text(encoding="utf-8")
+    frontmatter, body = _split_frontmatter(content)
+    lines = body.strip().split("\n")
+
+    h2_count = sum(1 for line in lines if re.match(r"^## ", line))
+    h3_count = sum(1 for line in lines if re.match(r"^### ", line))
+
+    code_blocks = re.findall(r"```(\w*)", content)
+    code_block_languages = [lang for lang in code_blocks if lang]
+
+    lower_body = body.lower()
+    has_examples = bool(re.search(r"(## example|### example|## usage)", lower_body))
+    has_troubleshooting = bool(re.search(r"(## troubleshoot|## common issue|## faq)", lower_body))
+
+    refs_dir = skill_dir / "references"
+    assets_dir = skill_dir / "assets"
+    reference_files = (
+        [f.name for f in refs_dir.iterdir() if f.is_file()] if refs_dir.exists() else []
+    )
+    asset_files = (
+        [f.name for f in assets_dir.iterdir() if f.is_file()] if assets_dir.exists() else []
+    )
+
+    total_lines = len(content.split("\n"))
+    for ref_file in reference_files:
+        ref_path = refs_dir / ref_file
+        total_lines += len(ref_path.read_text(encoding="utf-8").split("\n"))
+
+    must_pattern = re.compile(r"\b(MUST|NEVER|ALWAYS)\b")
+    must_count = len(must_pattern.findall(content))
+
+    cross_refs = re.findall(r"(?:skill|skills)/([a-z0-9-]+)", body)
+
+    return ParsedSkill(
+        path=skill_dir,
+        name=frontmatter.get("name", skill_dir.name),
+        description=frontmatter.get("description", ""),
+        line_count=len(content.split("\n")),
+        h2_count=h2_count,
+        h3_count=h3_count,
+        code_block_count=len(code_blocks),
+        code_block_languages=code_block_languages,
+        has_examples=has_examples,
+        has_troubleshooting=has_troubleshooting,
+        has_references=refs_dir.exists(),
+        has_assets=assets_dir.exists(),
+        reference_files=reference_files,
+        asset_files=asset_files,
+        total_content_lines=total_lines,
+        must_never_always_count=must_count,
+        cross_references=cross_refs,
+        raw_content=content,
+        frontmatter=frontmatter,
+    )
+
+
+def parse_agent(agent_path: Path) -> ParsedAgent:
+    """Parse an agent markdown file."""
+    content = agent_path.read_text(encoding="utf-8")
+    frontmatter, body = _split_frontmatter(content)
+
+    tools_str = frontmatter.get("tools", "")
+    tools = [t.strip() for t in tools_str.split(",")] if tools_str else []
+
+    description = frontmatter.get("description", "")
+    has_proactive = bool(re.search(r"use proactively", description, re.IGNORECASE))
+
+    skill_refs = re.findall(r"(?:skill|skills)/([a-z0-9-]+)", body)
+
+    return ParsedAgent(
+        path=agent_path,
+        name=frontmatter.get("name", agent_path.stem),
+        description=description,
+        model=frontmatter.get("model"),
+        has_tools_restriction=bool(tools),
+        tools=tools,
+        has_proactive_trigger=has_proactive,
+        skill_references=skill_refs,
+        raw_content=content,
+        frontmatter=frontmatter,
+    )
+
+
+def parse_plugin(plugin_dir: Path) -> ParsedPlugin:
+    """Parse an entire plugin directory."""
+    plugin_json_path = plugin_dir / ".claude-plugin" / "plugin.json"
+    plugin_json = {}
+    if plugin_json_path.exists():
+        plugin_json = json.loads(plugin_json_path.read_text(encoding="utf-8"))
+
+    name = plugin_json.get("name", plugin_dir.name)
+
+    skills = []
+    skills_dir = plugin_dir / "skills"
+    if skills_dir.exists():
+        for skill_dir in sorted(skills_dir.iterdir()):
+            if skill_dir.is_dir() and (skill_dir / "SKILL.md").exists():
+                skills.append(parse_skill(skill_dir))
+
+    agents = []
+    agents_dir = plugin_dir / "agents"
+    if agents_dir.exists():
+        for agent_file in sorted(agents_dir.glob("*.md")):
+            agents.append(parse_agent(agent_file))
+
+    return ParsedPlugin(
+        path=plugin_dir,
+        name=name,
+        skills=skills,
+        agents=agents,
+        plugin_json=plugin_json,
+    )
+
+
+def _split_frontmatter(content: str) -> tuple[dict, str]:
+    """Split YAML frontmatter from markdown body."""
+    if not content.startswith("---"):
+        return {}, content
+
+    parts = content.split("---", 2)
+    if len(parts) < 3:
+        return {}, content
+
+    try:
+        frontmatter = yaml.safe_load(parts[1]) or {}
+    except yaml.YAMLError:
+        frontmatter = {}
+
+    return frontmatter, parts[2]

plugins/plugin-eval/tests/test_parser.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.parser import ParsedSkill, ParsedAgent, ParsedPlugin, parse_skill, parse_agent, parse_plugin
+
+
+class TestParseSkill:
+    def test_parse_valid_skill(self, sample_skill_dir: Path):
+        skill = parse_skill(sample_skill_dir)
+        assert skill.name == "test-skill"
+        assert "testing plugin-eval" in skill.description
+        assert skill.line_count > 0
+        assert skill.h2_count >= 2
+        assert skill.code_block_count >= 1
+        assert skill.has_references is True
+
+    def test_parse_poor_skill(self, poor_skill_dir: Path):
+        skill = parse_skill(poor_skill_dir)
+        assert skill.name == "poor-skill"
+        assert skill.must_never_always_count > 15
+        assert skill.has_references is True
+        assert len(skill.reference_files) == 1
+
+    def test_missing_skill_md_raises(self, tmp_path: Path):
+        empty = tmp_path / "empty-skill"
+        empty.mkdir()
+        with pytest.raises(FileNotFoundError):
+            parse_skill(empty)
+
+
+class TestParseAgent:
+    def test_parse_valid_agent(self, sample_plugin_dir: Path):
+        agent_path = sample_plugin_dir / "agents" / "test-agent.md"
+        agent = parse_agent(agent_path)
+        assert agent.name == "test-agent"
+        assert agent.model == "sonnet"
+        assert agent.has_tools_restriction is True
+        assert agent.has_proactive_trigger is True
+
+
+class TestParsePlugin:
+    def test_parse_valid_plugin(self, sample_plugin_dir: Path):
+        plugin = parse_plugin(sample_plugin_dir)
+        assert plugin.name == "test-plugin"
+        assert len(plugin.skills) == 1
+        assert len(plugin.agents) == 1