From af2efbfecf68d91de052d393dfa80bc965c7232a Mon Sep 17 00:00:00 2001 From: lennyzeltser Date: Mon, 9 Feb 2026 23:47:38 -0500 Subject: [PATCH] feat: expand tool coverage and surface catalog extras in suggest_tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tool definitions for redress (Go binary analysis), uncompyle6 (Python decompiler), pyinstxtractor-ng, and apkid (Android packer detection). Update pyinstxtractor preprocessor to prefer -ng variant with fallback. suggest_tools now queries the tool catalog and returns additional_tools for the detected file type, deduplicated against the registry via .py normalization and an alias map (e.g., oletools → oleid/olevba/rtfobj). Catalog loading is now graceful — a missing or corrupt tools-index.json logs a warning instead of crashing the server. New category mappings for Go→ELF and Python. Updated analysis hints for redress, uncompyle6, apkid. --- src/catalog/__tests__/catalog.test.ts | 24 +++ src/catalog/index.ts | 24 ++- src/handlers/__tests__/suggest-tools.test.ts | 151 +++++++++++++++++++ src/handlers/suggest-tools.ts | 68 ++++++++- src/tools/definitions.ts | 45 ++++++ src/tools/preprocessors.ts | 9 +- 6 files changed, 311 insertions(+), 10 deletions(-) diff --git a/src/catalog/__tests__/catalog.test.ts b/src/catalog/__tests__/catalog.test.ts index 074e4a0..a6d0074 100644 --- a/src/catalog/__tests__/catalog.test.ts +++ b/src/catalog/__tests__/catalog.test.ts @@ -50,4 +50,28 @@ describe("ToolCatalog", () => { it("returns empty array for unknown salt-states category", () => { expect(toolCatalog.forSaltCategory("Fake Category")).toEqual([]); }); + + // ── New category mappings (Python, Go→ELF, Android→APK) ────────────── + + it("maps 'Statically Analyze Code: Python' to MCP Python category", () => { + const pythonTools = toolCatalog.forMcpCategory("Python"); + const saltCats = new Set(pythonTools.map((t) => t.category)); + expect(saltCats).toContain("Statically Analyze Code: Python"); + }); + + it("maps Go salt-states category to ELF when tools exist", () => { + // "Examine Static Properties: Go" is mapped to ELF in SALT_TO_MCP_CATEGORY. + // Once update-docs.py syncs redress into tools-index.json, Go tools will + // appear in forMcpCategory("ELF"). For now verify ELF includes its base categories. + const elfTools = toolCatalog.forMcpCategory("ELF"); + expect(elfTools.length).toBeGreaterThan(0); + const saltCats = new Set(elfTools.map((t) => t.category)); + expect(saltCats).toContain("Examine Static Properties: ELF Files"); + }); + + it("maps 'Statically Analyze Code: Android' to MCP APK category", () => { + const apkTools = toolCatalog.forMcpCategory("APK"); + const saltCats = new Set(apkTools.map((t) => t.category)); + expect(saltCats).toContain("Statically Analyze Code: Android"); + }); }); diff --git a/src/catalog/index.ts b/src/catalog/index.ts index bc8e784..01746a3 100644 --- a/src/catalog/index.ts +++ b/src/catalog/index.ts @@ -43,10 +43,14 @@ const SALT_TO_MCP_CATEGORY: Record = { // Office (OLE2, OOXML, RTF share tools) "Analyze Documents: Microsoft Office": "OLE2", - // ELF + // ELF / Go "Examine Static Properties: ELF Files": "ELF", + "Examine Static Properties: Go": "ELF", "Dynamically Reverse-Engineer Code: ELF Files": "ELF", + // Python + "Statically Analyze Code: Python": "Python", + // Scripts "Statically Analyze Code: Scripts": "Script", "Dynamically Reverse-Engineer Code: Scripts": "Script", @@ -87,9 +91,9 @@ class ToolCatalog { readonly updated: string; constructor(index: ToolsIndex) { - this.tools = index.tools; - this.version = index.version; - this.updated = index.updated; + this.tools = index.tools ?? []; + this.version = index.version ?? "0.0.0"; + this.updated = index.updated ?? "unknown"; // Index by salt-states category this.bySaltCategory = new Map(); @@ -147,5 +151,15 @@ function loadIndex(): ToolsIndex { return JSON.parse(raw) as ToolsIndex; } +/** Load index with graceful fallback so a missing/corrupt file doesn't crash the server. */ +function loadIndexSafe(): ToolsIndex { + try { + return loadIndex(); + } catch (err) { + console.error("WARNING: Failed to load tool catalog — additional_tools will be unavailable:", err); + return { version: "0.0.0", updated: "unknown", tools: [] }; + } +} + /** Singleton catalog instance. */ -export const toolCatalog = new ToolCatalog(loadIndex()); +export const toolCatalog = new ToolCatalog(loadIndexSafe()); diff --git a/src/handlers/__tests__/suggest-tools.test.ts b/src/handlers/__tests__/suggest-tools.test.ts index 7d4a17f..ea000be 100644 --- a/src/handlers/__tests__/suggest-tools.test.ts +++ b/src/handlers/__tests__/suggest-tools.test.ts @@ -1,5 +1,7 @@ import { describe, it, expect, vi } from "vitest"; import { handleSuggestTools } from "../suggest-tools.js"; +import { toolCatalog } from "../../catalog/index.js"; +import { toolRegistry } from "../../tools/registry.js"; import { createMockDeps, ok, parseEnvelope } from "./helpers.js"; describe("handleSuggestTools", () => { @@ -68,4 +70,153 @@ describe("handleSuggestTools", () => { const env = parseEnvelope(result); expect(env.success).toBe(false); }); + + it("includes additional_tools from catalog for PE files", async () => { + const deps = createMockDeps(); + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.exe: PE32 executable (GUI) Intel 80386") + ); + + // Catalog should have PE tools beyond the registry + const catalogPeTools = toolCatalog.forMcpCategory("PE"); + expect(catalogPeTools.length).toBeGreaterThan(0); + + const result = await handleSuggestTools(deps, { file: "test.exe" }); + const env = parseEnvelope(result); + expect(env.success).toBe(true); + // additional_tools should be present (catalog has tools beyond registry) + expect(env.data.additional_tools).toBeDefined(); + expect(Array.isArray(env.data.additional_tools)).toBe(true); + { + // Each entry should have required fields + for (const tool of env.data.additional_tools) { + expect(tool).toHaveProperty("command"); + expect(tool).toHaveProperty("name"); + expect(tool).toHaveProperty("description"); + } + } + }); + + it("deduplicates .py-suffixed tools between registry and catalog", async () => { + const deps = createMockDeps(); + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.exe: PE32 executable (GUI) Intel 80386") + ); + + const result = await handleSuggestTools(deps, { file: "test.exe" }); + const env = parseEnvelope(result); + expect(env.success).toBe(true); + + // Build registry command set the same way the handler does + const normalize = (c: string) => c.replace(/\.py$/, ''); + const registryCommands = new Set( + toolRegistry.byTagAndTier("pe", "standard").map((t) => normalize(t.command)) + ); + const additionalCommands = (env.data.additional_tools ?? []).map((t: { command: string }) => t.command); + + // No normalized overlap between registry commands and additional_tools + for (const cmd of additionalCommands) { + expect(registryCommands.has(normalize(cmd))).toBe(false); + } + }); + + it("excludes catalog entries aliased to registry tools", async () => { + const deps = createMockDeps(); + + // PE file — readpe-formerly-pev should be excluded (aliased to pescan/pestr) + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.exe: PE32 executable (GUI) Intel 80386") + ); + const pe = await handleSuggestTools(deps, { file: "test.exe" }); + const peEnv = parseEnvelope(pe); + const peAdditional = (peEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + // Precondition: catalog must contain the aliased tool for the negative assertion to be meaningful + const peCatalog = toolCatalog.forMcpCategory("PE").map((t) => t.command); + expect(peCatalog).toContain("readpe-formerly-pev"); + expect(peAdditional).not.toContain("readpe-formerly-pev"); + + // PDF file — origamindee, pdftk-java should be excluded at standard depth; + // peepdf-3 should be excluded at deep depth (where peepdf enters the registry) + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/doc.pdf: PDF document, version 1.4") + ); + const pdf = await handleSuggestTools(deps, { file: "doc.pdf" }); + const pdfEnv = parseEnvelope(pdf); + const pdfAdditional = (pdfEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + // Precondition: catalog must contain the aliased tools + const pdfCatalog = toolCatalog.forMcpCategory("PDF").map((t) => t.command); + expect(pdfCatalog).toContain("origamindee"); + expect(pdfCatalog).toContain("pdftk-java"); + expect(pdfAdditional).not.toContain("origamindee"); + expect(pdfAdditional).not.toContain("pdftk-java"); + // peepdf-3 should still appear at standard depth (peepdf is deep-tier only) + expect(pdfAdditional).toContain("peepdf-3"); + + // At deep depth, peepdf enters registry so peepdf-3 alias kicks in + const pdfDeep = await handleSuggestTools(deps, { file: "doc.pdf", depth: "deep" }); + const pdfDeepEnv = parseEnvelope(pdfDeep); + const pdfDeepAdditional = (pdfDeepEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + expect(pdfDeepAdditional).not.toContain("peepdf-3"); + + // OLE2 file — oletools, xlmmacrodeobfuscator should be excluded + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/doc.doc: Composite Document File V2 Document") + ); + const ole = await handleSuggestTools(deps, { file: "doc.doc" }); + const oleEnv = parseEnvelope(ole); + const oleAdditional = (oleEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + // Precondition: catalog must contain the aliased tools + const oleCatalog = toolCatalog.forMcpCategory("OLE2").map((t) => t.command); + expect(oleCatalog).toContain("oletools"); + expect(oleCatalog).toContain("xlmmacrodeobfuscator"); + expect(oleAdditional).not.toContain("oletools"); + expect(oleAdditional).not.toContain("xlmmacrodeobfuscator"); + + // Python file — decompyle and pyinstaller-extractor should be excluded (aliased) + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.pyc: python 3.9 byte-compiled") + ); + const py = await handleSuggestTools(deps, { file: "test.pyc" }); + const pyEnv = parseEnvelope(py); + const pyAdditional = (pyEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + expect(pyAdditional).not.toContain("decompyle"); + expect(pyAdditional).not.toContain("pyinstaller-extractor"); + + // DOTNET file — ilspy should be excluded (aliased to ilspycmd) + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.exe: PE32 executable (console) Intel 80386 Mono/.Net assembly") + ); + const dotnet = await handleSuggestTools(deps, { file: "test.exe" }); + const dotnetEnv = parseEnvelope(dotnet); + const dotnetAdditional = (dotnetEnv.data.additional_tools ?? []).map((t: { command: string }) => t.command); + expect(dotnetAdditional).not.toContain("ilspy"); + }); + + it("has no duplicates within additional_tools", async () => { + const deps = createMockDeps(); + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.exe: PE32 executable (GUI) Intel 80386") + ); + + const result = await handleSuggestTools(deps, { file: "test.exe" }); + const env = parseEnvelope(result); + if (!env.data.additional_tools) return; + + const commands = env.data.additional_tools.map((t: { command: string }) => t.command); + expect(commands.length).toBe(new Set(commands).size); + }); + + it("omits additional_tools when catalog has no extras", async () => { + const deps = createMockDeps(); + // Use a category unlikely to have catalog extras + vi.mocked(deps.connector.execute).mockResolvedValue( + ok("/samples/test.one: Microsoft OneNote") + ); + + const result = await handleSuggestTools(deps, { file: "test.one" }); + const env = parseEnvelope(result); + expect(env.success).toBe(true); + // OneNote has no catalog mapping, so additional_tools should be absent + expect(env.data.additional_tools).toBeUndefined(); + }); }); diff --git a/src/handlers/suggest-tools.ts b/src/handlers/suggest-tools.ts index 20690f2..8895417 100644 --- a/src/handlers/suggest-tools.ts +++ b/src/handlers/suggest-tools.ts @@ -4,10 +4,29 @@ import { validateFilePath } from "../security/blocklist.js"; import { matchFileType, CATEGORY_TAG_MAP } from "../file-type-mappings.js"; import type { DepthTier } from "../file-type-mappings.js"; import { toolRegistry } from "../tools/registry.js"; +import { toolCatalog } from "../catalog/index.js"; import { formatResponse, formatError } from "../response.js"; import { REMnuxError } from "../errors/remnux-error.js"; import { toREMnuxError } from "../errors/error-mapper.js"; +/** + * Maps catalog command identifiers to their corresponding registry commands. + * The catalog uses salt-states package names while the registry uses actual + * CLI commands. One catalog entry may correspond to multiple registry tools + * when a package ships several binaries. + */ +const CATALOG_ALIASES: Record = { + "readpe-formerly-pev": ["pescan", "pestr"], + "ilspy": ["ilspycmd"], + "origamindee": ["pdfcop", "pdfextract", "pdfdecompress"], + "pdftk-java": ["pdftk"], + "peepdf-3": ["peepdf"], + "xlmmacrodeobfuscator": ["xlmdeobfuscator"], + "oletools": ["oleid", "olevba", "rtfobj"], + "decompyle": ["pycdc"], + "pyinstaller-extractor": ["pyinstxtractor-ng"], +}; + /** * Base per-category expert guidance for the AI agent. */ @@ -29,6 +48,7 @@ const BASE_HINTS: Record = { "yara-forge scans for malware family signatures from 45+ sources (Malpedia, ReversingLabs). Matches indicate resemblance to known families, not confirmed attribution. " + "yara-rules provides supplementary capability/behavior detection (packers, anti-debug). " + "1768.py analyzes Cobalt Strike beacons. disitool.py examines Authenticode signatures. " + + "redress recovers Go compiler info, packages, and types from Go-compiled executables — returns minimal output on non-Go PE files. " + "For deep analysis, capa -vv shows matched rule details with addresses. " + "pedump shows raw PE structure and brxor bruteforces XOR-encoded strings.", PDF: @@ -63,6 +83,7 @@ const BASE_HINTS: Record = { "readelf -h shows ELF header (type, arch, entry point). " + "readelf -S lists sections — look for unusual section names or sizes. " + "capa detects capabilities in ELF binaries similar to PE analysis. " + + "redress analyzes Go binaries — recovers package names, types, compiler version, and source structure. On non-Go ELF files it returns minimal output (just OS/arch). " + "For deep analysis, capa -vv shows matched rule details with addresses.", JavaScript: "js-beautify reformats and deobfuscates JavaScript — look for eval(), " + @@ -81,8 +102,8 @@ const BASE_HINTS: Record = { "translate.py applies byte-level transforms. numbers-to-string.py decodes numeric payloads.", Python: "pycdc decompiles Python bytecode (.pyc) to readable source code. " + - "For PyInstaller bundles (which produce PE files), use pyinstxtractor via run_tool " + - "to extract contents, then analyze the resulting .pyc files with pycdc.", + "uncompyle6 is an alternative decompiler supporting Python 1.0 through 3.8 — use when pycdc fails or for older Python versions. " + + "For PyInstaller bundles, pyinstxtractor-ng extracts contents without requiring a matching Python version — invoke via run_tool.", JAR: "zipdump lists the JAR archive contents (JAR files are ZIP format). " + "Look for unusual class files, embedded resources, or manifest entries. " + @@ -93,6 +114,7 @@ const BASE_HINTS: Record = { "Look for notable attachments, embedded URLs, and header anomalies. " + "Extract attachments for further analysis with appropriate tools.", APK: + "apkid identifies compilers, packers, and obfuscators — use it for triage alongside droidlysis to understand what protections are applied. " + "apktool decompiles the APK to smali and extracts resources. " + "droidlysis performs static analysis identifying permissions, API calls, and risk indicators. " + "Look for excessive permissions, obfuscation, and notable network activity.", @@ -286,6 +308,42 @@ export async function handleSuggestTools( ...(availableCommands.has(t.command) ? {} : { available: false as const }), })); + // Query catalog for additional tools not in the registry + let additionalTools: Array<{ command: string; name: string; description: string; website: string }> = []; + try { + const normalize = (c: string) => c.replace(/\.py$/, ''); + const registryCommands = new Set(tools.map((t) => normalize(t.command))); + + // Check if a catalog tool is already covered by the registry + const isCovered = (catalogCmd: string): boolean => { + // Direct match (with .py normalization) + if (registryCommands.has(normalize(catalogCmd))) return true; + // Alias match (catalog package name → registry CLI commands) + // Alias match: catalog package name → registry CLI commands. + // Uses some() deliberately: if ANY tool from the package is recommended, + // the catalog package entry is hidden to avoid confusing overlap. + const aliases = CATALOG_ALIASES[catalogCmd]; + return aliases !== undefined && aliases.some((a) => registryCommands.has(normalize(a))); + }; + + const seen = new Set(); + additionalTools = toolCatalog.forMcpCategory(category.name) + .filter((ct) => !isCovered(ct.command)) + .filter((ct) => { + if (seen.has(ct.command)) return false; + seen.add(ct.command); + return true; + }) + .map((ct) => ({ + command: ct.command, + name: ct.name, + description: ct.description, + website: ct.website, + })); + } catch (err) { + console.error("WARNING: Catalog query failed for additional_tools:", err); + } + return formatResponse("suggest_tools", { file: args.file, detected_type: fileOutput, @@ -296,6 +354,12 @@ export async function handleSuggestTools( warning: `No tools registered for category "${category.name}" at depth "${depth}". Try depth "deep" or use run_tool directly.`, }), analysis_hints: generateHints(category.name, fileOutput), + ...(additionalTools.length > 0 && { + additional_tools: additionalTools, + additional_tools_note: + "Additional tools available on REMnux for this file type. " + + "Not auto-run by analyze_file. Use run_tool to invoke manually.", + }), }, startTime); } catch (error) { return formatError("suggest_tools", toREMnuxError(error, deps.config.mode), startTime); diff --git a/src/tools/definitions.ts b/src/tools/definitions.ts index 9a0d66b..c802a50 100644 --- a/src/tools/definitions.ts +++ b/src/tools/definitions.ts @@ -524,6 +524,19 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [ tier: "standard", }, + // ── Go binary analysis ───────────────────────────────────────────────── + { + name: "redress", + description: "Analyze Go binaries to recover package names, type definitions, source structure, and compiler version.", + command: "redress", + inputStyle: "positional", + fixedArgs: ["info"], + outputFormat: "text", + timeout: 60, + tags: ["elf", "pe"], + tier: "standard", + }, + // ── ELF analysis ──────────────────────────────────────────────────────── { name: "readelf-header", @@ -639,6 +652,27 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [ tags: ["python"], tier: "standard", }, + { + name: "uncompyle6", + description: "Decompile Python bytecode (.pyc) to source code. Supports Python 1.0 through 3.8.", + command: "uncompyle6", + inputStyle: "positional", + outputFormat: "text", + timeout: 120, + tags: ["python", "decompilation"], + tier: "deep", + }, + { + name: "pyinstxtractor-ng", + description: "Extract contents of PyInstaller executables without requiring a matching Python version.", + command: "pyinstxtractor-ng", + inputStyle: "positional", + outputFormat: "text", + timeout: 60, + tags: ["pe", "python"], + tier: "standard", + requiresUserArgs: true, // Extracts to filesystem; better used via run_tool + }, // ── JAR ───────────────────────────────────────────────────────────────── // zipdump already covers JAR (tagged "jar") @@ -689,6 +723,17 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [ }, // ── APK ───────────────────────────────────────────────────────────────── + { + name: "apkid", + description: "Identify compilers, packers, and obfuscators used to protect Android APK and DEX files.", + command: "apkid", + inputStyle: "positional", + fixedArgs: ["-j"], + outputFormat: "json", + timeout: 120, + tags: ["apk", "packer-detection"], + tier: "standard", + }, { name: "apktool", description: "Reverse-engineer Android APK files.", diff --git a/src/tools/preprocessors.ts b/src/tools/preprocessors.ts index 68de7f5..47d0cbe 100644 --- a/src/tools/preprocessors.ts +++ b/src/tools/preprocessors.ts @@ -56,14 +56,17 @@ export const PREPROCESSORS: Preprocessor[] = [ timeout: 60000, }, { - name: "pyinstxtractor", + name: "pyinstxtractor-ng", description: "Extract files from PyInstaller bundles", categories: ["PE"], // Check for PyInstaller magic bytes at end of file detectCommand: (fp) => `python3 -c "import sys; f=open(sys.argv[1],'rb'); f.seek(-24,2); exit(0 if f.read(13)==b'MEI\\x0c\\x0b\\x0a\\x0b\\x0e' else 1)" ${shellEscape(fp)}`, - processCommand: (fp, out) => - `pyinstxtractor ${shellEscape(fp)} -d ${shellEscape(out)}`, + processCommand: (fp, out) => { + const basename = fp.split('/').pop() ?? fp; + return `(pyinstxtractor-ng ${shellEscape(fp)} && mv ${shellEscape(basename + '_extracted')} ${shellEscape(out)}) || ` + + `(pyinstxtractor.py ${shellEscape(fp)} && mv ${shellEscape(basename + '_extracted')} ${shellEscape(out)})`; + }, timeout: 60000, }, ];