From c17af8acc22c900f8726ddc5c5f670c3b6c35e77 Mon Sep 17 00:00:00 2001 From: clearbluejar <3752074+clearbluejar@users.noreply.github.com> Date: Thu, 6 Feb 2025 20:24:52 +0000 Subject: [PATCH] pyhidra -> pyghidra --- .devcontainer/devcontainer.json | 2 +- .devcontainer/post-create.sh | 4 +-- .../workflows/pytest-devcontainer-pypi.yml | 4 ++- .../pytest-devcontainer-repo-all.yml | 4 ++- .github/workflows/pytest-devcontainer.yml | 2 +- .github/workflows/pytest-docker.yml | 6 ++-- ghidriff/correlators.py | 7 ++-- ghidriff/ghidra_diff_engine.py | 20 +++++------ ghidriff/utils.py | 2 +- tests/test_ghidra_zip_format_import.py | 23 ++++++------- tests/test_import.py | 33 ++++++++++--------- tests/test_startup.py | 32 +++++++++--------- www/docs/ghidriff.md | 2 +- 13 files changed, 74 insertions(+), 67 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9389807..38ed0c1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ { "name": "ghidriff", // image from https://github.com/clearbluejar/ghidra-python - "image": "ghcr.io/clearbluejar/ghidra-python:11.2.1ghidra3.12python-bookworm", + "image": "ghcr.io/clearbluejar/ghidra-python:11.3ghidra3.12python-bookworm", // Configure tool-specific properties. "customizations": { // Configure properties specific to VS Code. diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index fc332e1..2e03090 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -24,8 +24,8 @@ fi # install local workspace and test requirements pip install -e ".[testing]" -# initialize pyhidra -python -m pyhidra.install_plugins +# initialize pyghidra +python -m pyghidra.install_plugins # git clone test data if dir doesn't exist TEST_DATA_PATH="tests/data" diff --git a/.github/workflows/pytest-devcontainer-pypi.yml b/.github/workflows/pytest-devcontainer-pypi.yml index 954e7f6..c1ba9f9 100644 --- a/.github/workflows/pytest-devcontainer-pypi.yml +++ b/.github/workflows/pytest-devcontainer-pypi.yml @@ -22,6 +22,8 @@ jobs: # cover the latest and all versions of all subreleases image: [ "latest", + "11.3ghidra3.12python-bookworm", + "11.2.1ghidra3.10python-bookworm", "11.1.2ghidra3.12python-bookworm", "11.0.3ghidra3.11python-bookworm", "10.4ghidra3.11python-bookworm", @@ -57,7 +59,7 @@ jobs: pip install "ghidriff[testing]" pip list # install plugins before use - python -m pyhidra.install_plugins + python -m pyghidra.install_plugins # download data to shared test data if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi pytest -rA -n auto \ No newline at end of file diff --git a/.github/workflows/pytest-devcontainer-repo-all.yml b/.github/workflows/pytest-devcontainer-repo-all.yml index b202b94..6068de8 100644 --- a/.github/workflows/pytest-devcontainer-repo-all.yml +++ b/.github/workflows/pytest-devcontainer-repo-all.yml @@ -26,6 +26,8 @@ jobs: # cover the latest and all versions of all subreleases image: [ "latest", + "11.3ghidra3.12python-bookworm", + "11.2.1ghidra3.10python-bookworm" "11.1.1ghidra3.12python-bookworm", "11.0.3ghidra3.11python-bookworm", "10.4ghidra3.11python-bookworm", @@ -60,7 +62,7 @@ jobs: # install package and testing pip install -e ".[testing]" # install plugins before use - python -m pyhidra.install_plugins + python -m pyghidra.install_plugins # download data to shared test data if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi pytest -rA -n auto \ No newline at end of file diff --git a/.github/workflows/pytest-devcontainer.yml b/.github/workflows/pytest-devcontainer.yml index f294a54..16f4c91 100644 --- a/.github/workflows/pytest-devcontainer.yml +++ b/.github/workflows/pytest-devcontainer.yml @@ -36,7 +36,7 @@ jobs: # install package and testing pip install -e ".[testing]" # install plugins before use - python -m pyhidra.install_plugins + python -m pyghidra.install_plugins # download data to shared test data if [ ! -d "tests/data" ]; then git clone https://github.com/clearbluejar/ghidriff-test-data.git tests/data; fi pytest -rA -n auto \ No newline at end of file diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index b82f633..261f65f 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -54,12 +54,12 @@ jobs: # install ghidriff package and testing reqs pip install ".[testing]" ls -R /root - #pyhidra & + #pyghidra & #sleep 10 #killall python # source .env/bin/activate - # #init pyhidra - python -m pyhidra.install_plugins + # #init pyghidra + python -m pyghidra.install_plugins # pushd /tmp # popd # download data to shared test data diff --git a/ghidriff/correlators.py b/ghidriff/correlators.py index a4b4bf7..b431470 100644 --- a/ghidriff/correlators.py +++ b/ghidriff/correlators.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: import ghidra from ghidra_builtins import * + @JImplements(FunctionHasher, deferred=True) class StructuralGraphHasher: """ @@ -347,7 +348,7 @@ def get_defined_data(program: "ghidra.program.model.listing.Program"): # its a string, find which functions use it for ref in sym.references: # print(ref.referenceType.toString()) - f = program.getFunctionManager().getFunctionContaining(ref.fromAddress) + f = program.getFunctionManager().getFunctionContaining(ref.getFromAddress()) if f is not None: func_str_map.setdefault(f.entryPoint, []).append(str(data)) @@ -452,7 +453,7 @@ def get_func_to_switch(program: "ghidra.program.model.listing.Program"): # if f is None: # for ref in sym.references: # # print(ref.referenceType.toString()) - # f = program.getFunctionManager().getFunctionContaining(ref.fromAddress) + # f = program.getFunctionManager().getFunctionContaining(ref.getFromAddress()) # if f is not None: # func_switch_map.setdefault(f.entryPoint, []).append(str(sym)) # else: @@ -461,7 +462,7 @@ def get_func_to_switch(program: "ghidra.program.model.listing.Program"): func_switch_map.setdefault(f.entryPoint, []).append(str(sym)) for ref in sym.references: # print(ref.referenceType.toString()) - f = program.getFunctionManager().getFunctionContaining(ref.fromAddress) + f = program.getFunctionManager().getFunctionContaining(ref.getFromAddress()) if f is not None: func_switch_map.setdefault(f.entryPoint, []).append(str(sym)) diff --git a/ghidriff/ghidra_diff_engine.py b/ghidriff/ghidra_diff_engine.py index 26cb83d..02cb045 100644 --- a/ghidriff/ghidra_diff_engine.py +++ b/ghidriff/ghidra_diff_engine.py @@ -12,7 +12,7 @@ from typing import List, Tuple, Union, TYPE_CHECKING from argparse import Namespace import logging -from pyhidra.launcher import PyhidraLauncher +from pyghidra.launcher import PyGhidraLauncher from .utils import sha1_file, get_microsoft_download_url, get_pe_extra_data from .markdown import GhidriffMarkdown @@ -25,10 +25,10 @@ if TYPE_CHECKING: from ghidra_builtins import * -class HeadlessLoggingPyhidraLauncher(PyhidraLauncher): +class HeadlessLoggingPyGhidraLauncher(PyGhidraLauncher): """ - Headless pyhidra launcher - Slightly Modified from Pyhidra to allow the Ghidra log path to be set + Headless pyghidra launcher + Slightly Modified from PyGhidra to allow the Ghidra log path to be set """ def __init__(self, verbose=False, log_path=None): @@ -36,7 +36,7 @@ class HeadlessLoggingPyhidraLauncher(PyhidraLauncher): self.log_path = log_path def _launch(self): - from pyhidra.launcher import _silence_java_output + from pyghidra.launcher import _silence_java_output from ghidra.framework import Application, HeadlessGhidraApplicationConfiguration from java.io import File with _silence_java_output(not self.verbose, not self.verbose): @@ -89,8 +89,8 @@ class GhidraDiffEngine(GhidriffMarkdown, metaclass=ABCMeta): else: self.logger.warn('Engine File Log: {engine_log_path}') - # Init Pyhidra - launcher = HeadlessLoggingPyhidraLauncher(verbose=verbose, log_path=engine_log_path) + # Init PyGhidra + launcher = HeadlessLoggingPyGhidraLauncher(verbose=verbose, log_path=engine_log_path) # JVM Settings @@ -301,8 +301,8 @@ class GhidraDiffEngine(GhidriffMarkdown, metaclass=ABCMeta): ref_types = set() for ref in sym.references: - ref_types.add(ref.referenceType.toString()) - f = prog.getFunctionManager().getFunctionContaining(ref.fromAddress) + ref_types.add(ref.getReferenceType().toString()) + f = prog.getFunctionManager().getFunctionContaining(ref.getFromAddress()) if f: calling.add(f.getName()) @@ -333,7 +333,7 @@ class GhidraDiffEngine(GhidriffMarkdown, metaclass=ABCMeta): # instruction and mnemonic bulker for code in code_units: instructions.append(str(code)) - mnemonics.append(str(code.mnemonicString)) + mnemonics.append(str(code.getMnemonicString)) from ghidra.program.model.block import BasicBlockModel diff --git a/ghidriff/utils.py b/ghidriff/utils.py index 86cdf91..5f24634 100644 --- a/ghidriff/utils.py +++ b/ghidriff/utils.py @@ -67,7 +67,7 @@ def get_microsoft_download_url(filename, timestamp, virtual_size): return f'https://msdl.microsoft.com/download/symbols/{filename}/{timestamp}{virtual_size}/{filename}' -# utils from Pyhidra +# utils from PyGhidra def get_private_class(path: str): from java.lang import ClassLoader diff --git a/tests/test_ghidra_zip_format_import.py b/tests/test_ghidra_zip_format_import.py index d2fefb2..cb49935 100644 --- a/tests/test_ghidra_zip_format_import.py +++ b/tests/test_ghidra_zip_format_import.py @@ -1,7 +1,7 @@ from pathlib import Path import json import pytest -from pyhidra import HeadlessPyhidraLauncher +from pyghidra import HeadlessPyGhidraLauncher from ghidriff import get_parser, VersionTrackingDiff, GhidraDiffEngine @@ -18,8 +18,8 @@ def test_diff_afd_cve_2023_21768_gzf(shared_datadir: Path): """ # check ghidra version and bail if old - - if HeadlessPyhidraLauncher().app_info.version < '11.0': + + if HeadlessPyGhidraLauncher().app_info.version < '11.0': # gzf files were made with 11.0 print('Skip testing gzf on < 11.0') return @@ -29,10 +29,10 @@ def test_diff_afd_cve_2023_21768_gzf(shared_datadir: Path): output_path.mkdir(exist_ok=True, parents=True) symbols_path = shared_datadir / SYMBOLS_DIR bins_path = shared_datadir / BINS_DIR - bins_path = shared_datadir / BINS_DIR + bins_path = shared_datadir / BINS_DIR ghidra_project_path = output_path / 'ghidra_projects' - ghidra_project_path.mkdir(exist_ok=True,parents=True) - + ghidra_project_path.mkdir(exist_ok=True, parents=True) + # setup bins old_bin_path = bins_path / 'afd.sys.x64.10.0.22621.1028.gzf' new_bin_path = bins_path / 'afd.sys.x64.10.0.22621.1415.gzf' @@ -44,7 +44,8 @@ def test_diff_afd_cve_2023_21768_gzf(shared_datadir: Path): GhidraDiffEngine.add_ghidra_args_to_parser(parser) - args = parser.parse_args(['-s', str(symbols_path), str(old_bin_path.absolute()), str(new_bin_path.absolute()), '-p', str(ghidra_project_path.absolute())]) + args = parser.parse_args(['-s', str(symbols_path), str(old_bin_path.absolute()), + str(new_bin_path.absolute()), '-p', str(ghidra_project_path.absolute())]) engine_log_path = output_path / parser.get_default('log_path') @@ -110,7 +111,7 @@ def test_diff_afd_cve_2023_21768_gzf_with_one_nongzf(shared_datadir: Path): """ # check ghidra version and bail if old - if HeadlessPyhidraLauncher().app_info.version < '11.0': + if HeadlessPyGhidraLauncher().app_info.version < '11.0': # gzf files were made with 11.0 print('Skip testing gzf on < 11.0') return @@ -121,8 +122,7 @@ def test_diff_afd_cve_2023_21768_gzf_with_one_nongzf(shared_datadir: Path): symbols_path = shared_datadir / SYMBOLS_DIR bins_path = shared_datadir / BINS_DIR ghidra_project_path = output_path / 'ghidra_projects' - ghidra_project_path.mkdir(exist_ok=True,parents=True) - + ghidra_project_path.mkdir(exist_ok=True, parents=True) # setup bins @@ -136,7 +136,8 @@ def test_diff_afd_cve_2023_21768_gzf_with_one_nongzf(shared_datadir: Path): GhidraDiffEngine.add_ghidra_args_to_parser(parser) - args = parser.parse_args(['-s', str(symbols_path), str(old_bin_path.absolute()), str(new_bin_path.absolute()), '-p', str(ghidra_project_path.absolute())]) + args = parser.parse_args(['-s', str(symbols_path), str(old_bin_path.absolute()), + str(new_bin_path.absolute()), '-p', str(ghidra_project_path.absolute())]) engine_log_path = output_path / parser.get_default('log_path') diff --git a/tests/test_import.py b/tests/test_import.py index fbbb0dd..ab6d769 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -1,7 +1,7 @@ from pathlib import Path import json import pytest -from pyhidra import HeadlessPyhidraLauncher +from pyghidra import HeadlessPyGhidraLauncher from ghidriff import get_parser, get_engine_classes, VersionTrackingDiff, GhidraDiffEngine @@ -17,7 +17,7 @@ def test_gzf_import_program(shared_datadir: Path): Tests that gzf files contain expected programs """ - if HeadlessPyhidraLauncher().app_info.version < '11.0': + if HeadlessPyGhidraLauncher().app_info.version < '11.0': # gzf files were made with 11.0 print('Skip testing gzf on < 11.0') return @@ -28,15 +28,16 @@ def test_gzf_import_program(shared_datadir: Path): symbols_path = shared_datadir / SYMBOLS_DIR bins_path = shared_datadir / BINS_DIR ghidra_project_path = output_path / 'ghidra_projects' - ghidra_project_path.mkdir(exist_ok=True,parents=True) + ghidra_project_path.mkdir(exist_ok=True, parents=True) # bins bins_to_import = [ # bin path , expected program - ['afd.sys.x64.10.0.22621.1028', 'afd.sys.x64.10.0.22621.1028-00a2b7'], #if a gzf file is used first, this becomes really unstable... + # if a gzf file is used first, this becomes really unstable... + ['afd.sys.x64.10.0.22621.1028', 'afd.sys.x64.10.0.22621.1028-00a2b7'], ['afd.sys.x64.10.0.22621.1415', 'afd.sys.x64.10.0.22621.1415-095200'], - ['afd.sys.x64.10.0.22621.1028.gzf', 'afd.sys.x64.10.0.22621.1028.gzf-338a92'], - ['afd.sys.x64.10.0.22621.1415.gzf', 'afd.sys.x64.10.0.22621.1415.gzf-fc498a'], + ['afd.sys.x64.10.0.22621.1028.gzf', 'afd.sys.x64.10.0.22621.1028.gzf-338a92'], + ['afd.sys.x64.10.0.22621.1415.gzf', 'afd.sys.x64.10.0.22621.1415.gzf-fc498a'], ['ntoskrnl.exe.x64.10.0.22621.2792.10-1-5.gzf', 'ntoskrnl.exe.x64.10.0.22621.2792.10-1-5.gzf-acb020'], ['ntoskrnl.exe.x64.10.0.22621.2861.10-1-5.gzf', 'ntoskrnl.exe.x64.10.0.22621.2861.10-1-5.gzf-0e4e43'], ] @@ -46,13 +47,14 @@ def test_gzf_import_program(shared_datadir: Path): GhidraDiffEngine.add_ghidra_args_to_parser(parser) engine_log_path = output_path / parser.get_default('log_path') - - binary_paths = [path for path in [bins_path / name[0] for name in bins_to_import ]] - - args = parser.parse_args(['-s', str(symbols_path),'test', 'test2', '-p', str(ghidra_project_path.absolute())]) # these args will not be tested - expected_names = [name for name in [name[1] for name in bins_to_import ]] - + binary_paths = [path for path in [bins_path / name[0] for name in bins_to_import]] + + args = parser.parse_args(['-s', str(symbols_path), 'test', 'test2', '-p', + str(ghidra_project_path.absolute())]) # these args will not be tested + + expected_names = [name for name in [name[1] for name in bins_to_import]] + binary_paths = [Path(path) for path in binary_paths] if any([not path.exists() for path in binary_paths]): @@ -62,7 +64,7 @@ def test_gzf_import_program(shared_datadir: Path): import uuid # ensure fresh test each time project_name = f'import-test-{uuid.uuid4()}' - #project_name = f'import-test' + # project_name = f'import-test' DiffEngine: GhidraDiffEngine = VersionTrackingDiff @@ -89,8 +91,7 @@ def test_gzf_import_program(shared_datadir: Path): # print(data) # assert expected_names[i] == data[0] - for i,import_path in enumerate(binary_paths): + for i, import_path in enumerate(binary_paths): imports_result = d.setup_project([binary_paths[i]], args.project_location, project_name, args.symbols_path) - #d.project.wait() + # d.project.wait() assert expected_names[i] == imports_result[0][0] - diff --git a/tests/test_startup.py b/tests/test_startup.py index 4307c36..972524d 100644 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -4,9 +4,9 @@ from pytest import MonkeyPatch import pytest -def test_pyhidra_start(): - import pyhidra - pyhidra.start(verbose=True) +def test_pyghidra_start(): + import pyghidra + pyghidra.start(verbose=True) def test_ghidra_install_dir(): @@ -19,31 +19,31 @@ def test_ghidra_install_dir(): # def setup_bogus_env(mon): -# @patch('pyhidra.GHIDRA_INSTALL_DIR', "/someboguspath"): +# @patch('pyghidra.GHIDRA_INSTALL_DIR', "/someboguspath"): # def test_bogus_ghidra_install_dira(): # with pytest.raises(FileNotFoundError): -# import pyhidra as err_pyhidra -# err_pyhidra.start(verbose=True) +# import pyghidra as err_pyghidra +# err_pyghidra.start(verbose=True) # def test_ghidra_install_dir(): # # import sys -# # sys.modules.pop('pyhidra') +# # sys.modules.pop('pyghidra') # with MonkeyPatch.context() as mp: # mp.delenv("GHIDRA_INSTALL_DIR") -# import pyhidra +# import pyghidra # # print(os.getenv("GHIDRA_INSTALL_DIR")) # with pytest.raises(SystemExit) as pytest_wrapped_e: -# launcher = pyhidra.start(verbose=True) +# launcher = pyghidra.start(verbose=True) # assert pytest_wrapped_e.type == SystemExit # #assert pytest_wrapped_e.value.code == 42 # import os # print(os.getenv("GHIDRA_INSTALL_DIR")) # # from importlib import reload -# # reload(pyhidra) +# # reload(pyghidra) # import sys -# del sys.modules['pyhidra'] +# del sys.modules['pyghidra'] # @pytest.mark.forked @@ -58,15 +58,15 @@ def test_ghidra_install_dir(): # launcher = None # # with pytest.raises(SystemExit): -# import pyhidra -# mp.setattr(pyhidra.constants, 'GHIDRA_INSTALL_DIR', '/someboguspath') -# print(pyhidra.constants) +# import pyghidra +# mp.setattr(pyghidra.constants, 'GHIDRA_INSTALL_DIR', '/someboguspath') +# print(pyghidra.constants) # print(os.getenv("GHIDRA_INSTALL_DIR")) -# launcher = pyhidra.start(verbose=True) +# launcher = pyghidra.start(verbose=True) # assert launcher == None - # launcher = pyhidra.start(verbose=True) + # launcher = pyghidra.start(verbose=True) # print(launcher.check_ghidra_version()) # det test_file_not_exist(): diff --git a/www/docs/ghidriff.md b/www/docs/ghidriff.md index d2dd7c7..03e3f6b 100644 --- a/www/docs/ghidriff.md +++ b/www/docs/ghidriff.md @@ -15,6 +15,6 @@ sidebar_position: 1 ## Ghidriff - Ghidra Binary Diffing Engine `ghidriff` provides a command-line binary diffing capability with a fresh take on diffing workflow and results. -It leverages the power of Ghidra's ProgramAPI and [FlatProgramAPI](https://ghidra.re/ghidra_docs/api/ghidra/program/flatapi/FlatProgramAPI.html) to find the *added*, *deleted*, and *modified* functions of two arbitrary binaries. It is written in Python3 using `pyhidra` to orchestrate Ghidra and `jpype` as the Python to Java interface to Ghidra. +It leverages the power of Ghidra's ProgramAPI and [FlatProgramAPI](https://ghidra.re/ghidra_docs/api/ghidra/program/flatapi/FlatProgramAPI.html) to find the *added*, *deleted*, and *modified* functions of two arbitrary binaries. It is written in Python3 using `pyghidra` to orchestrate Ghidra and `jpype` as the Python to Java interface to Ghidra. Its primary use case is patch diffing. Its ability to perform a patch diff with a single command makes it ideal for automated analysis. The diffing results are stored in JSON and rendered in markdown (optionally side-by-side HTML). The markdown output promotes "social" diffing, as results are easy to publish in a gist or include in your next writeup or blog post. \ No newline at end of file