From cc37bfdd292ce520ba1c44df7a3a70d5f8137236 Mon Sep 17 00:00:00 2001 From: Seth Hobson Date: Fri, 12 Jun 2026 10:55:25 +0900 Subject: [PATCH] feat: add fable model tier and refresh per-harness model aliases Add Claude Fable 5 (model: fable) as the tier above opus across the adapter framework - native passthrough in Claude Code, mapped to each harness's top model elsewhere. No agents are promoted yet; tier criteria and cost guidance are documented in docs/agents.md and docs/authoring.md. Refresh MODEL_ALIASES against current harness catalogs (June 2026): - Codex: gpt-5 family -> gpt-5.5 / gpt-5.4-mini (per Codex model docs) - Copilot: GPT-5 stand-ins -> native Claude models (claude-opus-4.8, claude-sonnet-4.6, claude-haiku-4.5) now that Copilot CLI serves them - OpenCode: opus -> anthropic/claude-opus-4-8; haiku dated pin -> latest-tracking anthropic/claude-haiku-4-5 - Gemini: intentionally stays on GA gemini-2.5-* (3.x is preview-gated) - plugin-eval judge opus tier -> claude-opus-4-8 Gates: generate-all (no committed drift), validate STRICT=1, garden, test (442 passed), smoke-test all green. --- AGENTS.md | 2 +- ARCHITECTURE.md | 2 +- README.md | 7 ++-- docs/agents.md | 25 ++++++++++++++ docs/authoring.md | 25 +++++++++++--- docs/harnesses.md | 5 +-- .../plugin_eval/layers/harness_portability.py | 2 +- .../src/plugin_eval/layers/judge.py | 2 +- tools/adapters/capabilities.py | 34 +++++++++++++------ tools/tests/test_adapters.py | 22 ++++++------ 10 files changed, 90 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ebf747e..bd803e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ Top-level `skills/` is Gemini output; do not use it for OpenCode installs. 192 subagents under `plugins/*/agents/.md`. Per-harness transpilation: -- **Codex**: `.codex/agents/__.toml` (drop `tools:`, map model alias to GPT-5 family, infer `sandbox_mode`) +- **Codex**: `.codex/agents/__.toml` (drop `tools:`, map model alias to the GPT-5.x family, infer `sandbox_mode`) - **OpenCode**: `.opencode/agents/__.md` with `mode: subagent` + `permission:` block (locked agents — those with source `tools: []` — get deny-everything except base `skill`/`task`) - **Gemini**: `agents/__.md` (April 2026 subagent spec) - **Cursor**: reads `.claude/agents/` directly diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 74e4d59..b3121b0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ Each adapter consumes the canonical `plugins/` source and emits harness-native a | `codex.py` | committed `.agents/plugins/marketplace.json` + `plugins/*/.codex-plugin/plugin.json`; gitignored `.codex/skills/`, `.codex/agents/*.toml` | Marketplace + per-plugin manifests (point at source `plugins/`); Markdown → TOML transform, 8 KB body cap with `references/` overflow, sandbox_mode heuristic, collision detection | | `cursor.py` | `.cursor-plugin/`, `.cursor/rules/*.mdc` | Marketplace manifests + hand-curated rules. Cursor reads `.claude/` directly for skills/agents | | `opencode.py` | `.opencode/agents/`, `.opencode/commands/`, `.opencode/skills/` | Permission block from `tools:` allowlist (locked agents preserve intent); strict lowercase tool names; OpenCode-safe skill names | -| `copilot.py` | `.copilot/agents/`, `.copilot/skills/`, `.copilot/commands/` | Markdown agent profiles + SKILL.md skills + commands-as-skills; model maps to GPT-5 family | +| `copilot.py` | `.copilot/agents/`, `.copilot/skills/`, `.copilot/commands/` | Markdown agent profiles + SKILL.md skills + commands-as-skills; model maps to native Claude models | | `gemini.py` | `skills/`, `agents/`, `commands/*.toml` at extension root | Native skills + April-2026 subagents; `@{path}` injection for large command bodies | Detail in [`docs/harnesses.md`](docs/harnesses.md) (capability matrix per harness) and [`docs/architecture.md`](docs/architecture.md) (full design rationale). diff --git a/README.md b/README.md index c92467d..b624b79 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,12 @@ plugins/python-development/ └── skills/ # 16 specialized skills (async, testing, packaging, …) ``` -Three-tier model strategy: +Tiered model strategy: | Tier | Model | Use | |---|---|---| -| 1 | Opus 4.7 | Architecture, security, code review, production-critical | +| 0 | Fable 5 | Longest-horizon autonomous work — large migrations, multi-hour runs (opt-in, premium cost) | +| 1 | Opus | Architecture, security, code review, production-critical | | 2 | inherit | User-chosen — backend, frontend, AI/ML, specialized | | 3 | Sonnet | Docs, testing, debugging, API references | | 4 | Haiku | Fast operational tasks, SEO, deployment, content | @@ -92,7 +93,7 @@ emits harness-native artifacts (not lowest-common-denominator translations): | **Cursor** | `.cursor-plugin/`, `.cursor/rules/` | Thin marketplace + curated rules; reuses `.claude/` | | **OpenCode** | `.opencode/agents/`, `.opencode/commands/`, `.opencode/skills/` | `permission:` block from `tools:` allowlist; OpenCode-safe skill names | | **Gemini CLI** | `skills/`, `agents/`, `commands/` (TOML) | Native skills + subagents (April 2026 spec) | -| **Copilot** | `.copilot/agents/`, `.copilot/skills/`, `.copilot/commands/` | Markdown agent profiles + SKILL.md skills + commands-as-skills; model maps to GPT-5 family | +| **Copilot** | `.copilot/agents/`, `.copilot/skills/`, `.copilot/commands/` | Markdown agent profiles + SKILL.md skills + commands-as-skills; model maps to native Claude models | ```bash make generate-all # all five diff --git a/docs/agents.md b/docs/agents.md index 769d35c..79135f3 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -230,6 +230,7 @@ Agents are assigned to specific Claude models based on task complexity and compu | Model | Agent Count | Use Case | | ------- | ----------- | --------------------------------------------------------------- | +| Fable | 0 | Longest-horizon autonomous work (tier above Opus; see criteria) | | Opus | 54 | Critical architecture, security, code review, production coding | | Sonnet | 62 | Complex tasks, support with intelligence | | Haiku | 20 | Fast operational tasks | @@ -263,6 +264,30 @@ Agents are assigned to specific Claude models based on task complexity and compu - Orchestrating multi-agent workflows - Handling business-critical legal/HR matters +#### Fable - Longest-Horizon Autonomous Work + +Claude Fable 5 (`model: fable`) is the tier above Opus. It is opt-in in Claude Code +(v2.1.170+, never the default) and carries roughly 2.6× the effective cost of Opus +($10/$50 per MTok plus a ~30% heavier tokenizer), so reserve it for agents where Opus +demonstrably needs multiple attempts. + +**Use when:** + +- Running multi-hour autonomous sessions (large codebase migrations, overnight refactors) +- Executing end-to-end work from a single well-specified goal +- Coordinating long-lived parallel sub-agent workstreams + +**Avoid for:** + +- Security-analysis agents — Fable 5's cyber/bio safety classifiers fall back to + Opus 4.8 on that content anyway +- Anything Sonnet or Opus already completes in one pass + +Other harnesses map `fable` to their top available model (see +[authoring.md](authoring.md#model-aliases)). Fable-tier agent bodies should state goals +and constraints rather than step-by-step scaffolding, and must not ask the model to echo +its internal reasoning. + ### Hybrid Orchestration Patterns The plugin ecosystem leverages Sonnet + Haiku orchestration for optimal performance and cost efficiency: diff --git a/docs/authoring.md b/docs/authoring.md index bb7f679..1b6779d 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -41,7 +41,7 @@ to invoke your skill/agent. ### Talk about actions, not tools -Codex's underlying GPT-5 doesn't have a `Read`/`Edit`/`Bash` vocabulary — the model picks +Codex's underlying GPT-5.x models don't have a `Read`/`Edit`/`Bash` vocabulary — the model picks the native tool from the action you describe. OpenCode is strict about lowercase (`read`, `bash`). Cursor's agent has its own vocabulary. @@ -112,14 +112,29 @@ clean naming — pick distinct names for skill/command pairs within a plugin. | Source field | Codex | Cursor | OpenCode | Gemini | Copilot | |---|---|---|---|---|---|---| -| `model: opus` | `gpt-5` | `inherit` | `anthropic/claude-opus-4-7` | `gemini-2.5-pro` | `gpt-5` | -| `model: sonnet` | `gpt-5-mini` | `inherit` | `anthropic/claude-sonnet-4-6` | `gemini-2.5-pro` | `gpt-5-mini` | -| `model: haiku` | `gpt-5-nano` | `inherit` | `anthropic/claude-haiku-4-5-20251001` | `gemini-2.5-flash` | `gpt-5-nano` | -| `model: inherit` | `gpt-5` | `inherit` | `anthropic/claude-sonnet-4-6` | `gemini-2.5-pro` | `gpt-5` | +| `model: fable` | `gpt-5.5` | `inherit` | `anthropic/claude-fable-5` | `gemini-2.5-pro` | `claude-opus-4.8` | +| `model: opus` | `gpt-5.5` | `inherit` | `anthropic/claude-opus-4-8` | `gemini-2.5-pro` | `claude-opus-4.8` | +| `model: sonnet` | `gpt-5.4-mini` | `inherit` | `anthropic/claude-sonnet-4-6` | `gemini-2.5-pro` | `claude-sonnet-4.6` | +| `model: haiku` | `gpt-5.4-mini` | `inherit` | `anthropic/claude-haiku-4-5` | `gemini-2.5-flash` | `claude-haiku-4.5` | +| `model: inherit` | `gpt-5.5` | `inherit` | `anthropic/claude-sonnet-4-6` | `gemini-2.5-pro` | `claude-sonnet-4.6` | The adapter handles mapping. The `BARE_MODEL_ALIAS` lint is informational — it just notes that the mapping is implicit. If you want explicit, use `inherit`. +Mapping targets live in `tools/adapters/capabilities.py` (`MODEL_ALIASES`) and track each +harness's published catalog (last verified June 2026). Copilot CLI serves Claude models +natively, so its aliases map Claude → Claude using Copilot's dotted IDs. Gemini stays on +the GA `gemini-2.5-*` family because Gemini 3.x ships only access-gated `-preview` IDs. + +`fable` (Claude Fable 5) is the tier above `opus`, reserved for the longest-horizon +autonomous work. It is native in Claude Code (v2.1.170+, opt-in, ~2.6× Opus effective +cost); other harnesses map it to their top available model. Tag an agent `fable` only +when Opus demonstrably needs multiple attempts at the task. Avoid it for +security-analysis agents — Fable 5's cyber/bio classifiers fall back to Opus there +anyway. Prefer stating goals over step-by-step scaffolding in fable-tier agent bodies, +and never instruct the model to echo its reasoning (triggers `reasoning_extraction` +refusals). + ## Skills layout for progressive disclosure The OpenAI harness-engineering post argues that "agents start with a small, stable entry diff --git a/docs/harnesses.md b/docs/harnesses.md index f4c6795..3a727c7 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -34,7 +34,7 @@ as Claude Code markdown. Per-harness artifacts are generated by adapters under ` | Context file recommended cap | 150 lines / 500 tokens | 150 lines / 500 tokens | 150 lines / 500 tokens | 150 lines / 500 tokens | 150 lines / 500 tokens | | Skill body hard cap | none | **8 KB** | none | none | none | | Tool name case | CamelCase (`Read`) | action verbs (no tool vocab) | lowercase | lowercase (strict) | lowercase (Gemini names) | -| Bare model aliases | ✅ (`opus`/`sonnet`/`haiku`) | mapped to GPT-5 family | use `inherit` | full provider/model-id | mapped to `gemini-2.5-*` | +| Bare model aliases | ✅ (`fable`/`opus`/`sonnet`/`haiku`) | mapped to GPT-5.x family | use `inherit` | full provider/model-id | mapped to `gemini-2.5-*` | ## Claude Code native features @@ -60,7 +60,8 @@ rules to write portable content. |---|---|---|---|---| | `tools: Read, Grep` (agent allowlist) | dropped; `sandbox_mode = "read-only"` heuristic | dropped (Cursor doesn't honor) | converted to `permission:` deny block | passed through | | `color: blue` (agent) | dropped | dropped | dropped | dropped | -| `model: opus` (agent) | mapped to `gpt-5` | rewritten to `inherit` | rewritten to `anthropic/claude-opus-4-7` | mapped to `gemini-2.5-pro` | +| `model: opus` (agent) | mapped to `gpt-5.5` | rewritten to `inherit` | rewritten to `anthropic/claude-opus-4-8` | mapped to `gemini-2.5-pro` | +| `model: fable` (agent) | mapped to `gpt-5.5` | rewritten to `inherit` | rewritten to `anthropic/claude-fable-5` | mapped to `gemini-2.5-pro` | | `TodoWrite` in body | no equivalent — leave as-is | no equivalent — leave as-is | works as-is | no equivalent | | Skill body > 8 KB | split into `references/details.md` | passed through | passed through | passed through | | Agent named `worker` | namespaced to `__worker` | passed through | passed through | passed through | diff --git a/plugins/plugin-eval/src/plugin_eval/layers/harness_portability.py b/plugins/plugin-eval/src/plugin_eval/layers/harness_portability.py index 9cc5b70..7f06866 100644 --- a/plugins/plugin-eval/src/plugin_eval/layers/harness_portability.py +++ b/plugins/plugin-eval/src/plugin_eval/layers/harness_portability.py @@ -54,7 +54,7 @@ _CAMEL_TOOL_PATTERN = re.compile( _TOOL_PROSE_PATTERN = re.compile(rf"(?i:\bthe)\s+(?:`)?({_TOOL_ALTERNATION})(?:`)?\s+tool\b") # Bare model aliases that don't map cleanly (Cursor/OpenCode/Gemini need full IDs) -_BARE_MODEL_ALIAS_PATTERN = re.compile(r"^(opus|sonnet|haiku)$") +_BARE_MODEL_ALIAS_PATTERN = re.compile(r"^(fable|opus|sonnet|haiku)$") # Context file line cap (per harness-engineering principle) _CONTEXT_FILE_LINE_CAP = 150 diff --git a/plugins/plugin-eval/src/plugin_eval/layers/judge.py b/plugins/plugin-eval/src/plugin_eval/layers/judge.py index a64b630..c4b8c5d 100644 --- a/plugins/plugin-eval/src/plugin_eval/layers/judge.py +++ b/plugins/plugin-eval/src/plugin_eval/layers/judge.py @@ -38,7 +38,7 @@ Score 1.0 — Perfectly calibrated: Minimal surface area, maximum cohesion, idea _MODEL_MAP: dict[str, str] = { "haiku": "claude-haiku-4-5-20251001", "sonnet": "claude-sonnet-4-6", - "opus": "claude-opus-4-7", + "opus": "claude-opus-4-8", } diff --git a/tools/adapters/capabilities.py b/tools/adapters/capabilities.py index cdf65dc..0c8f735 100644 --- a/tools/adapters/capabilities.py +++ b/tools/adapters/capabilities.py @@ -244,39 +244,51 @@ TOOL_NAME_MAPS: dict[str, dict[str, str]] = { } -# Model alias map: bare Claude alias -> full provider-prefixed ID per harness +# Model alias map: bare Claude alias -> full provider-prefixed ID per harness. +# Targets verified against each harness's published model catalog 2026-06: +# Codex recommends gpt-5.5 (top) / gpt-5.4-mini (light tasks, subagents); +# Copilot CLI serves Claude models natively, so aliases map Claude -> Claude +# (no Fable in Copilot — top available Claude is Opus 4.8, dotted-ID format); +# Gemini CLI's GA models remain gemini-2.5-* (3.x is preview-gated and its +# IDs churn), so the gemini column intentionally stays on the 2.5 family. MODEL_ALIASES: dict[str, dict[str, str]] = { "claude-code": { + "fable": "fable", "opus": "opus", "sonnet": "sonnet", "haiku": "haiku", "inherit": "inherit", }, "codex": { - "opus": "gpt-5", - "sonnet": "gpt-5-mini", - "haiku": "gpt-5-nano", - "inherit": "gpt-5", + "fable": "gpt-5.5", + "opus": "gpt-5.5", + "sonnet": "gpt-5.4-mini", + "haiku": "gpt-5.4-mini", + "inherit": "gpt-5.5", }, "copilot": { - "opus": "gpt-5", - "sonnet": "gpt-5-mini", - "haiku": "gpt-5-nano", - "inherit": "gpt-5", + "fable": "claude-opus-4.8", + "opus": "claude-opus-4.8", + "sonnet": "claude-sonnet-4.6", + "haiku": "claude-haiku-4.5", + "inherit": "claude-sonnet-4.6", }, "cursor": { + "fable": "inherit", "opus": "inherit", "sonnet": "inherit", "haiku": "inherit", "inherit": "inherit", }, "opencode": { - "opus": "anthropic/claude-opus-4-7", + "fable": "anthropic/claude-fable-5", + "opus": "anthropic/claude-opus-4-8", "sonnet": "anthropic/claude-sonnet-4-6", - "haiku": "anthropic/claude-haiku-4-5-20251001", + "haiku": "anthropic/claude-haiku-4-5", "inherit": "anthropic/claude-sonnet-4-6", }, "gemini": { + "fable": "gemini-2.5-pro", "opus": "gemini-2.5-pro", "sonnet": "gemini-2.5-pro", "haiku": "gemini-2.5-flash", diff --git a/tools/tests/test_adapters.py b/tools/tests/test_adapters.py index 6821b8c..96d8585 100644 --- a/tools/tests/test_adapters.py +++ b/tools/tests/test_adapters.py @@ -76,8 +76,8 @@ class TestCodexAdapter: parsed = tomllib.loads(content) assert parsed["name"] == "demo__greeter" - # opus is mapped to gpt-5 - assert parsed["model"] == "gpt-5" + # opus is mapped to gpt-5.5 + assert parsed["model"] == "gpt-5.5" # tools: Read, Grep -> read-only-ish set -> sandbox_mode = read-only assert parsed["sandbox_mode"] == "read-only" # color: blue should NOT be present @@ -561,7 +561,7 @@ class TestOpenCodeAdapter: assert fm["name"] == "demo__greeter" assert fm["mode"] == "subagent" # opus -> full provider/model-id - assert fm["model"] == "anthropic/claude-opus-4-7" + assert fm["model"] == "anthropic/claude-opus-4-8" def test_permission_block_denies_unlisted_tools( self, synthetic_plugin: PluginSource, output_root: Path @@ -945,7 +945,7 @@ class TestCopilotAdapter: fm, body = parse_frontmatter(agent_path.read_text()) assert fm["name"] == "demo__greeter" assert fm["description"] == "Use when delegating greetings." - assert fm["model"] == "gpt-5" + assert fm["model"] == "claude-opus-4.8" assert fm["tools"] == ["read", "search"] assert "color" not in fm @@ -1014,10 +1014,10 @@ class TestCopilotAdapter: CopilotAdapter(output_root=output_root).emit_plugin(plugin) expected = { - "sonnet-agent": "gpt-5-mini", - "haiku-agent": "gpt-5-nano", - "inherit-agent": "gpt-5", - "default-model": "gpt-5", + "sonnet-agent": "claude-sonnet-4.6", + "haiku-agent": "claude-haiku-4.5", + "inherit-agent": "claude-sonnet-4.6", + "default-model": "claude-sonnet-4.6", } for name, exp_model in expected.items(): fm, _ = parse_frontmatter( @@ -1099,7 +1099,7 @@ class TestCopilotAdapter: fm, body = parse_frontmatter(content) assert fm["name"] == "demo__advisory" assert fm["description"] == "Use when advising." - assert fm["model"] == "gpt-5-mini" + assert fm["model"] == "claude-sonnet-4.6" assert "tools:" in content def test_no_tools_field(self, tmp_path: Path, output_root: Path): @@ -1124,7 +1124,7 @@ class TestCopilotAdapter: fm, body = parse_frontmatter(content) assert fm["name"] == "demo__unrestricted" assert fm["description"] == "Use when unrestricted." - assert fm["model"] == "gpt-5" + assert fm["model"] == "claude-opus-4.8" assert "tools" not in fm @@ -1200,5 +1200,5 @@ class TestCapabilities: from tools.adapters.capabilities import MODEL_ALIASES, supported_harnesses for harness in supported_harnesses(): - for alias in ("opus", "sonnet", "haiku", "inherit"): + for alias in ("fable", "opus", "sonnet", "haiku", "inherit"): assert alias in MODEL_ALIASES[harness], f"{harness} missing {alias}"