admin/clearbluejar-ghidriff
1
0
Fork 0
mirror of https://github.com/clearbluejar/ghidriff synced 2026-06-08 13:32:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refresh dev environment workflow

Browse Source
This commit is contained in:
clearbluejar
2026-05-10 04:52:07 +00:00
parent 53e16cf2a1
commit 93b86db933
10 changed files with 233 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devcontainer/devcontainer.json
+8 -19
View File
@@ -3,7 +3,7 @@
{
"name": "ghidriff",
// image from https://github.com/clearbluejar/ghidra-python
"image": "ghcr.io/clearbluejar/ghidra-python:11.4ghidra3.12python-bookworm",
"image": "ghcr.io/clearbluejar/ghidra-python:12.0.4ghidra3.13python-bookworm",
// Configure tool-specific properties.
"customizations": {
// Configure properties specific to VS Code.
@@ -11,33 +11,22 @@
// Set *default* container specific settings.json values on container create.
"settings": {
"python.defaultInterpreterPath": "/usr/local/bin/python",
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
// VS code settings for ghidra-stubs autocomplete
"python.analysis.stubPath": "${workspaceFolder}/.env/lib/python3.12/site-packages/ghidra-stubs/",
"python.autoComplete.extraPaths": [
"${workspaceFolder}/.env/lib/python3.12/site-packages/ghidra-stubs/"
],
"python.terminal.activateEnvironment": true,
// ghidra-stubs installs outside normal import paths; post-create links this to the active venv.
"python.analysis.extraPaths": [
"${workspaceFolder}/.env/lib/python3.12/site-packages/ghidra-stubs/"
"${workspaceFolder}/.env/ghidra-stubs"
],
"[python]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "charliermarsh.ruff",
},
"ruff.nativeServer": "on",
},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"charliermarsh.ruff",
"yzhang.markdown-all-in-one"
]
}
@@ -51,4 +40,4 @@
],
// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode",
}
}
.devcontainer/post-create.sh
+15 -6
View File
@@ -5,9 +5,6 @@ source .env/bin/activate
# upgrade pip
pip install --upgrade pip
# Download latest pyi typings for Ghidra Version
pip install ghidra-stubs
# If arm64 os, need to build native binaries for Ghidra
if uname -a | grep -q 'aarch64'; then
if [ -e $GHIDRA_INSTALL_DIR/support/buildNatives ]
@@ -21,9 +18,21 @@ if uname -a | grep -q 'aarch64'; then
fi
fi
# install local workspace and test requirements
pip install -e ".[testing]"
# install local workspace, test requirements, and dev tooling
pip install -e ".[testing,dev]"
# Link ghidra-stubs to a stable path for VS Code/Pylance autocomplete.
STUB_PATH="$(python - <<'PY'
from pathlib import Path
import sysconfig
stub_path = Path(sysconfig.get_paths()["purelib"]) / "ghidra-stubs"
print(stub_path if stub_path.exists() else "")
PY
)"
if [ -n "$STUB_PATH" ]; then
ln -sfn "$STUB_PATH" .env/ghidra-stubs
fi
# git clone test data if dir doesn't exist
TEST_DATA_PATH="tests/data"
@@ -48,4 +57,4 @@ python tests/init_pyghidra.py
# popd
# echo 'To open up a Ghidra latest dev: code ~/ghidra-master'
# echo 'To open up a Ghidra latest dev: code ~/ghidra-master'
.gitignore
+2 -1
View File
@@ -133,8 +133,9 @@ dmypy.json
.ghidra_bridge*/
.symbols*/
ghidriffs/
binaries/
# pytest data (pulled from https://github.com/clearbluejar/ghidriff-test-data)
tests/data
.DS_Store
.DS_Store
.vscode/settings.json
Vendored
+13 -15
View File
@@ -1,18 +1,16 @@
{
// needed to make ghidra stubs work in this project (auto complete for vscode)
"python.defaultInterpreterPath": "${workspaceFolder}/.env/bin/python",
"python.analysis.stubPath": "${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/",
"python.autoComplete.extraPaths": [
"${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/"
],
"python.analysis.extraPaths": [
"${workspaceFolder}/.env/lib/python3.11/site-packages/ghidra-stubs/"
],
"python.defaultInterpreterPath": "${workspaceFolder}/.env/bin/python",
"python.terminal.activateEnvironment": true,
// ghidra-stubs installs outside normal import paths; post-create links this to the active venv.
"python.analysis.extraPaths": [
"${workspaceFolder}/.env/ghidra-stubs"
],
// env vars
"GHIDRA_INSTALL_DIR": "${env:GHIDRA_INSTALL_DIR}",
"sarif-viewer.connectToGithubCodeScanning": "off",
"liveServer.settings.port": 5501,
"[python]": {
"editor.defaultFormatter": "ms-python.autopep8"
},
}
"sarif-viewer.connectToGithubCodeScanning": "off",
"liveServer.settings.port": 5501,
"[python]": {
"editor.defaultFormatter": "charliermarsh.ruff"
},
"ruff.nativeServer": "on",
}
Makefile
+36
View File
@@ -0,0 +1,36 @@
PYTHON ?= $(shell test -x .env/bin/python && echo .env/bin/python || echo python)
PIP ?= $(PYTHON) -m pip
PYTEST ?= $(PYTHON) -m pytest
RUFF ?= $(PYTHON) -m ruff
.PHONY: install install-dev dev-setup test test-fast test-integration lint format clean check
install:
$(PIP) install -e .
install-dev:
$(PIP) install -e ".[testing,dev]"
dev-setup: install-dev
$(PYTHON) tests/init_pyghidra.py
test:
$(PYTEST)
test-fast:
$(PYTEST) -m fast
test-integration:
$(PYTEST) -m integration
lint:
$(RUFF) check ghidriff tests
format:
$(RUFF) format ghidriff tests
check: lint test-fast
clean:
rm -rf build dist *.egg-info .pytest_cache .ruff_cache
find ghidriff tests -type d -name __pycache__ -prune -exec rm -rf {} +
README.md
+57 -26
View File
@@ -138,13 +138,13 @@ Each implementation leverages the base class, and implements `find_changes`.
## Usage
```bash
usage: ghidriff [-h] [--engine {SimpleDiff,StructualGraphDiff,VersionTrackingDiff}] [-o OUTPUT_PATH] [--summary SUMMARY] [-p PROJECT_LOCATION]
usage: ghidriff [-h] [--engine {SimpleDiff,StructualGraphDiff,VersionTrackingDiff}] [-o OUTPUT_PATH] [--summary] [-p PROJECT_LOCATION]
[-n PROJECT_NAME] [-s SYMBOLS_PATH] [-g GZFS_PATH] [--ba BASE_ADDRESS] [--program-options PROGRAM_OPTIONS] [--threaded | --no-threaded]
[--force-analysis] [--force-diff] [--no-symbols] [--log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}]
[--file-log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}] [--log-path LOG_PATH] [--va] [--min-func-len MIN_FUNC_LEN]
[--use-calling-counts | --no-use-calling-counts] [--gdt GDT] [--bsim | --no-bsim] [--bsim-full | --no-bsim-full]
[--max-ram-percent MAX_RAM_PERCENT] [--print-flags] [--jvm-args [JVM_ARGS]] [--sxs] [--max-section-funcs MAX_SECTION_FUNCS]
[--md-title MD_TITLE]
[--max-ram-percent MAX_RAM_PERCENT] [--print-flags] [--jvm-args JVM_ARGS] [--decompiler-timeout DECOMPILER_TIMEOUT] [--sxs]
[--max-section-funcs MAX_SECTION_FUNCS] [--md-title MD_TITLE]
old new [new ...]
ghidriff - A Command Line Ghidra Binary Diffing Engine
@@ -159,7 +159,7 @@ options:
The diff implementation to use. (default: VersionTrackingDiff)
-o OUTPUT_PATH, --output-path OUTPUT_PATH
Output path for resulting diffs (default: ghidriffs)
--summary SUMMARY Add a summary diff if more than two bins are provided (default: False)
--summary Add a summary diff if more than two bins are provided (default: False)
```
@@ -213,8 +213,10 @@ JVM Options:
--max-ram-percent MAX_RAM_PERCENT
Set JVM Max Ram % of host RAM (default: 60.0)
--print-flags Print JVM flags at start (default: False)
--jvm-args [JVM_ARGS]
JVM args to add at start (default: None)
--jvm-args [JVM_ARGS]
JVM arg to add at start. Repeat as needed; use --jvm-args=-Xmx8G for values beginning with "-". (default: [])
--decompiler-timeout DECOMPILER_TIMEOUT
Decompiler timeout in seconds per function (default: 60)
Markdown Options:
--sxs Include side by side code diff (default: False)
@@ -277,11 +279,11 @@ If you are reverse engineering firmware or other fun binary and want to change t
$ ghidriff --base-address 0x80000 STM32F103C-firmware.bin STM32F103Ca-firmware.bin
```
## Quick Start Environment Setup
1. [Download](https://github.com/NationalSecurityAgency/ghidra/releases) and [install Ghidra](https://htmlpreview.github.io/?https://github.com/NationalSecurityAgency/ghidra/blob/stable/GhidraDocs/InstallationGuide.html#Install).
2. Set Ghidra Environment Variable `GHIDRA_INSTALL_DIR` to Ghidra install location.
3. Pip install `ghidriff`
## Quick Start Environment Setup
1. [Download](https://github.com/NationalSecurityAgency/ghidra/releases) and [install Ghidra](https://htmlpreview.github.io/?https://github.com/NationalSecurityAgency/ghidra/blob/stable/GhidraDocs/InstallationGuide.html#Install). The current development target is Ghidra 12.0.4 with pyghidra 3.x.
2. Set Ghidra Environment Variable `GHIDRA_INSTALL_DIR` to Ghidra install location.
3. Pip install `ghidriff`
### Windows
@@ -289,14 +291,22 @@ $ ghidriff --base-address 0x80000 STM32F103C-firmware.bin STM32F103Ca-firmware.b
PS C:\Users\user> [System.Environment]::SetEnvironmentVariable('GHIDRA_INSTALL_DIR','C:\ghidra_10.2.3_PUBLIC_20230208\ghidra_10.2.3_PUBLIC')
PS C:\Users\user> pip install ghidriff
```
### Linux / Mac
```bash
export GHIDRA_INSTALL_DIR="/path/to/ghidra/"
pip install ghidriff
```
### UV
### Linux / Mac
```bash
export GHIDRA_INSTALL_DIR="/path/to/ghidra/"
pip install ghidriff
```
On macOS, install a JDK supported by your Ghidra release first, then set `GHIDRA_INSTALL_DIR` to the unpacked Ghidra application directory. If macOS Gatekeeper quarantines the downloaded Ghidra archive, remove the quarantine attribute before first launch:
```bash
xattr -dr com.apple.quarantine /path/to/ghidra_12.0.4_PUBLIC
export GHIDRA_INSTALL_DIR="/path/to/ghidra_12.0.4_PUBLIC"
pip install ghidriff
```
### UV
```bash
export GHIDRA_INSTALL_DIR="/path/to/ghidra/"
@@ -356,12 +366,33 @@ ghidriffs
```
### Devcontainer - For Ghidriff development
Use the [.devcontainer](.devcontainer) in this repo. If you don't know how, follow the detailed instructions here: [ghidra-python-vscode-devcontainer-skeleton quick setup](https://github.com/clearbluejar/ghidra-python-vscode-devcontainer-skeleton#quick-start-setup---dev-container--best-option).
## Use Cases
### Devcontainer - For Ghidriff development
Use the [.devcontainer](.devcontainer) in this repo. If you don't know how, follow the detailed instructions here: [ghidra-python-vscode-devcontainer-skeleton quick setup](https://github.com/clearbluejar/ghidra-python-vscode-devcontainer-skeleton#quick-start-setup---dev-container--best-option).
The devcontainer targets `ghcr.io/clearbluejar/ghidra-python:12.0.4ghidra3.13python-bookworm`. After rebuilding it, the post-create step installs ghidriff with test and dev extras.
Useful development commands:
```bash
make install-dev
make test
make test-fast
make test-integration
make lint
make check
```
`make test` runs the full suite. `make test-fast` runs tests that do not launch Ghidra. `make test-integration` runs the Ghidra-backed tests and requires `tests/data`, `GHIDRA_INSTALL_DIR`, and a compatible pyghidra/Ghidra runtime. JVM arguments that begin with `-` should be passed with equals syntax, for example:
```bash
ghidriff --jvm-args=-Xmx8G --decompiler-timeout 120 old.bin new.bin
```
Deferred design items are tracked in [docs/deferred-issues.md](docs/deferred-issues.md).
## Use Cases
### Diffing a full Windows Kernel
@@ -806,4 +837,4 @@ Want to see the entire diff in a side by side? https://diffpreview.github.io/?f6
### Markdown Spec + MermaidJs
- Striving to be compliant with [GFM](https://github.github.com/gfm/) and [cmark](https://spec.commonmark.org/). Still working on it though. See issues.
- MermaidJs requires your markdown [renderer support](https://mermaid.js.org/ecosystem/integrations-community.html).
- MermaidJs requires your markdown [renderer support](https://mermaid.js.org/ecosystem/integrations-community.html).
docs/deferred-issues.md
+12
View File
@@ -0,0 +1,12 @@
# Deferred Issue Notes
These items are intentionally outside the current reliability refresh because they need design work, broader fixtures, or Ghidra-version-specific validation.
- Loader, language, and existing/manual project support: #129, #124, #114, and #130 should be designed together around explicit import planning rather than one-off CLI switches.
- Version Tracking session output/import: #135, #39, and #31 belong together as a VT interoperability feature. The current JSON/Markdown diff output remains the primary batch workflow.
- P-code correlator: #40 may be required for some small-function correctness cases, but it should be introduced as a new correlator with dedicated fixtures rather than folded into existing hash matching.
- Stack-frame diffing: #132 is a useful report enhancement once stable function stack metadata extraction is defined.
- Extensionless PE URL generation: #88 needs a reliable PE type/name heuristic for extensionless downloads before changing Microsoft symbol URL generation.
- Markdown linting: #55 should be handled after generated Markdown structure is made deterministic enough for GFM/cmark checks.
- Large-binary JVM stability and Java process exit behavior: #97 and #99 need reproducible cases and JVM/Ghidra-specific mitigation notes.
- Symbol porting, function categories, and pdiff dataclasses: #42, #15, and #5 are feature/refactor work and should not block reliability fixes.
docs/progress-log.md
+53
View File
@@ -0,0 +1,53 @@
# Reliability Refresh Progress Log
## Runtime
- Ghidra runtime checked locally: 12.0.4.
- pyghidra runtime checked locally: 3.0.2.
## Changed Files
- `.devcontainer/devcontainer.json`: target Ghidra 12.0.4/Python 3.13 image settings, stable ghidra-stubs path, and Ruff editor integration.
- `.devcontainer/post-create.sh`: install test/dev extras and link ghidra-stubs to a stable venv path.
- `.vscode/settings.json`: use a stable ghidra-stubs path and Ruff formatting.
- `setup.cfg`: require Python >=3.10, require pyghidra 3.x, add pytest markers and dev extra.
- `pyproject.toml`: add practical Ruff configuration.
- `Makefile`: add install, dev setup, fast/integration tests, lint, format, check, and clean targets.
- `ghidriff/parser.py`: make `--summary` a boolean flag.
- `ghidriff/__main__.py`: pass `--decompiler-timeout` into the engine.
- `ghidriff/ghidra_diff_engine.py`: add runtime compatibility warnings, typed CLI options, blocking decompiler queue, serialized analysis path, preflight checks, decompiler timeout use, stable `remove_code_sig` list return, small-function opt-in comparison path, unique program ID classification, and safer syntax/lint fixes.
- `ghidriff/version_tracking_diff.py`: guard invalid matches and fix `skip_types`.
- `tests/conftest.py`, `tests/test_fast_core.py`: split fast/integration tests and add fast regression coverage.
- `README.md`, `docs/deferred-issues.md`: document Ghidra 12.0.4/pyghidra 3.x workflow, Makefile commands, macOS notes, JVM arg syntax, integration-test path, and deferred design items.
- `.gitignore`: ignore local `binaries/`.
## Validation
- `make check PYTHON=.env/bin/python`: passed.
- `.env/bin/python -m pytest --collect-only -q`: passed, 26 tests collected.
- `.env/bin/python -m pytest tests/test_import.py::test_gzf_import_program -q`: passed.
- `.env/bin/python -m pytest tests/test_ghidra_zip_format_import.py::test_diff_afd_cve_2023_21768_gzf -q`: passed.
## Issues Addressed
- #138: Python support metadata now requires Python >=3.10.
- #137: `remove_code_sig` consistently returns a list of strings.
- #134: devcontainer and docs align to Ghidra 12.0.4 and pyghidra 3.x.
- #125: random analysis sleep replaced with a serialized analysis lock.
- #121: added an opt-in small-function comparison path and fast regression test.
- #119: added `--decompiler-timeout` and wired it through decompilation.
- #79: division-by-zero guard remains in stats calculation.
- #43: added Makefile/Ruff lint/dev workflow.
- #27: decompiler timeout is exposed as a user-facing decompiler option.
- #24: language and symbol-count preflight checks now raise actionable errors.
- #120: README macOS install notes updated for current Ghidra target.
## Deferred
Deferred design work is summarized in `docs/deferred-issues.md`: VT session import/export, loader/language/project planning, p-code correlator, stack-frame diffing, markdown linting, large-binary JVM stability, symbol porting, function categories, pdiff dataclasses, and extensionless PE URL heuristics.
## Residual Risks
- The full integration suite was not run end-to-end because the representative Ghidra tests already take minutes and some tests perform full analysis/diff flows.
- The small-function fix is intentionally conservative and opt-in through `--min-func-len`; broader correctness may still require #40 p-code correlator work.
- Ruff ignores existing broad style debt to provide a practical gate without a large unrelated formatting rewrite.
pyproject.toml
+26
View File
@@ -0,0 +1,26 @@
[tool.ruff]
line-length = 130
target-version = "py310"
extend-exclude = [
"ghidriff/basic-block-fix.py",
]
[tool.ruff.lint]
select = ["E", "F", "W"]
ignore = [
"E501",
"E711",
"E712",
"E722",
"F401",
"F403",
"F405",
"F541",
"F841",
"W291",
"W292",
"W293",
]
[tool.ruff.format]
quote-style = "single"
setup.cfg
+11 -4
View File
@@ -17,18 +17,18 @@ classifiers =
Intended Audience :: Developers
License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Programming Language :: Python :: 3
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: 3.12
Programming Language :: Python :: 3.13
[options]
python_requires = >= 3.9
python_requires = >= 3.10
packages = find:
zip_safe = False
include_package_data = True
install_requires =
pyghidra>=2.0.0
pyghidra>=3.0.0,<4
mdutils==1.6.0
[options.entry_points]
@@ -42,14 +42,21 @@ testing =
pytest-datadir
pytest-forked
pytest-xdist
dev =
ruff
ghidra-stubs
[tool:pytest]
testpaths = tests
required_plugins =
pytest-datadir
markers =
fast: tests that do not launch Ghidra
integration: tests that launch Ghidra or require analyzed Ghidra fixtures
slow: longer-running tests
addopts =
-p no:faulthandler
[pycodestyle]
max_line_length = 130
max_line_length = 130