admin/lifting-bits-sleigh
1
0
Fork 0
mirror of https://github.com/lifting-bits/sleigh synced 2026-06-21 13:56:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update auto-updater script

Browse Source 
Should now show changed files on each update.

Only updates if files that affect this project have changed
This commit is contained in:
Eric Kilmer
2022-12-15 13:03:08 -05:00
parent d3ca7d78d6
commit b80fefc001
2 changed files with 118 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/update_head.yml
+11 -2
View File
@@ -23,7 +23,7 @@ jobs:
id: head_update
run: |
# Sets some outputs. See next step
python3.7 scripts/update_ghidra_head.py --ci
python3 scripts/update_ghidra_head.py --ci
# Need this to run further Actions on the newly created PR
# See here for more details https://github.com/peter-evans/create-pull-request/blob/28fa4848947e0faa7fa50647691d01477589d5e9/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens
@@ -39,7 +39,16 @@ jobs:
uses: peter-evans/create-pull-request@v4
with:
title: Update Ghidra HEAD to commit ${{ steps.head_update.outputs.short_sha }}
commit-message: Bump Ghidra HEAD commit ${{ steps.head_update.outputs.short_sha }}
commit-message: |
Bump Ghidra HEAD commit ${{ steps.head_update.outputs.short_sha }}
Changed files:
${{ steps.head_update.outputs.changed_files }}
body: |
Changed files:
${{ steps.head_update.outputs.changed_files }}
branch: cron/update-ghidra-${{ steps.head_update.outputs.short_sha }}
delete-branch: true
token: ${{ steps.generate-token.outputs.token }}
scripts/update_ghidra_head.py
+107 -38
View File
@@ -1,18 +1,26 @@
"""Script to update CMake files for latest Ghidra Sleigh changes"""
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
from typing import AnyStr, Union
from pathlib import Path
from typing import AnyStr, Union, List
PROJECT_ROOT = pathlib.Path(__file__).parent.parent.resolve()
PROJECT_ROOT = Path(__file__).parent.parent.resolve()
HEAD_SPEC_FILE = PROJECT_ROOT / "src" / "spec_files_HEAD.cmake"
assert HEAD_SPEC_FILE.exists()
SETUP_GHIDRA_FILE = PROJECT_ROOT / "src" / "setup-ghidra-source.cmake"
assert SETUP_GHIDRA_FILE.exists()
# Paths in Ghidra repo that affect this repo. Used with git diff
SLEIGH_PATHS: List[str] = [
# Source code and tests
"Ghidra/Features/Decompiler/src/decompile",
# Sleigh files
"Ghidra/Processors",
]
GIT_EXE = shutil.which("git")
assert GIT_EXE is not None
@@ -21,21 +29,18 @@ def msg(s: str) -> None:
print(f"[!] {s}")
PathString = Union[AnyStr, pathlib.Path]
PathString = Union[AnyStr, Path]
def clone_ghidra_git(dir: PathString) -> None:
def clone_ghidra_git(clone_dir: PathString) -> None:
"""Clone the Ghidra git dir at specified directory"""
# Shallow clone
assert GIT_EXE is not None
subprocess.run(
[
GIT_EXE,
"clone",
"https://github.com/NationalSecurityAgency/ghidra",
"--depth",
"1",
dir,
clone_dir,
],
stdout=sys.stdout,
stderr=sys.stderr,
@@ -43,36 +48,90 @@ def clone_ghidra_git(dir: PathString) -> None:
)
def update_head_commit(latest_commit: str):
def git_get_changed_files(
repo: Path, old_commit: str, new_commit: str, paths: List[str], ci: bool
) -> List[str]:
"""Get list of changed files from old_commit to new_commit at the specified paths"""
assert GIT_EXE is not None
changed_files = (
subprocess.run(
[
GIT_EXE,
"diff",
"--name-status",
f"{old_commit}...{new_commit}",
"--",
*paths,
],
cwd=repo,
capture_output=True,
check=True,
)
.stdout.decode()
.strip()
.split("\n")
)
num_changed = len(changed_files)
if num_changed > 0:
msg(f"Found {num_changed} changed sleigh files:")
print("\n".join(changed_files))
if ci:
with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
gh_out.write("changed_files<<EOF\n")
gh_out.write("```\n")
gh_out.write("\n".join(changed_files))
gh_out.write("\n```\n")
gh_out.write("EOF\n")
return changed_files
def is_sleigh_updated(
ghidra_repo: Path, old_commit: str, new_commit: str, ci: bool
) -> bool:
"""Check if files we're interested in have been touched at all"""
changed_files = git_get_changed_files(
ghidra_repo, old_commit, new_commit, SLEIGH_PATHS, ci
)
return len(changed_files) > 0
def update_head_commit(
setup_file: Path, ghidra_repo_dir: PathString, latest_commit: str, ci: bool
) -> bool:
"""Edit the Ghidra script to point to the latest commit"""
head_commit_line = r"set\(ghidra_head_git_tag \"([0-9A-Fa-f]+)\"\)"
updated = False
fd, abspath = tempfile.mkstemp()
with open(fd, "w") as w:
with open(SETUP_GHIDRA_FILE, "r") as r:
with setup_file.open("r") as r:
for line in r:
match = re.search(head_commit_line, line)
if match is not None:
current_commit = match.group(1)
if current_commit != latest_commit:
msg(f"Found new commit: {latest_commit}")
line = re.sub(
head_commit_line,
f'set(ghidra_head_git_tag "{latest_commit}")',
line,
)
updated = True
if is_sleigh_updated(
ghidra_repo_dir, current_commit, latest_commit, ci
):
line = re.sub(
head_commit_line,
f'set(ghidra_head_git_tag "{latest_commit}")',
line,
)
updated = True
else:
msg("No sleigh files updated")
w.write(line)
# Make the swap with the new content
shutil.copymode(SETUP_GHIDRA_FILE, abspath)
os.remove(SETUP_GHIDRA_FILE)
shutil.move(abspath, SETUP_GHIDRA_FILE)
shutil.copymode(setup_file, abspath)
os.remove(setup_file)
shutil.move(abspath, setup_file)
return updated
def update_head_version_file(ghidra_root_dir: PathString) -> None:
def update_head_version_file(setup_file: Path, ghidra_root_dir: PathString) -> None:
"""Edit the Ghidra script to point to the latest version"""
cmake_head_version_line = (
r"set\(ghidra_head_version \"([0-9]+(\.[0-9]+)?(\.[0-9]+)?)\"\)"
@@ -86,7 +145,7 @@ def update_head_version_file(ghidra_root_dir: PathString) -> None:
assert match is not None
source_version = match.group(1)
with SETUP_GHIDRA_FILE.open("r") as f:
with setup_file.open("r") as f:
content = f.read()
match = re.search(cmake_head_version_line, content)
assert match is not None
@@ -99,7 +158,7 @@ def update_head_version_file(ghidra_root_dir: PathString) -> None:
msg(f"Found new version: {source_version}")
fd, abspath = tempfile.mkstemp()
with open(fd, "w") as w:
with SETUP_GHIDRA_FILE.open("r") as r:
with setup_file.open("r") as r:
for line in r:
match = re.search(cmake_head_version_line, line)
if match is not None:
@@ -111,9 +170,9 @@ def update_head_version_file(ghidra_root_dir: PathString) -> None:
w.write(line)
# Make the swap with the new content
shutil.copymode(SETUP_GHIDRA_FILE, abspath)
os.remove(SETUP_GHIDRA_FILE)
shutil.move(abspath, SETUP_GHIDRA_FILE)
shutil.copymode(setup_file, abspath)
os.remove(setup_file)
shutil.move(abspath, setup_file)
def update_spec_files(ghidra_repo_dir: PathString, cmake_file: PathString):
@@ -122,9 +181,7 @@ def update_spec_files(ghidra_repo_dir: PathString, cmake_file: PathString):
for dirpath, _, fnames in os.walk(ghidra_repo_dir / "Ghidra" / "Processors"):
for file in fnames:
if file.endswith(".slaspec"):
spec_files.append(
(pathlib.Path(dirpath) / file).relative_to(ghidra_repo_dir)
)
spec_files.append((Path(dirpath) / file).relative_to(ghidra_repo_dir))
assert len(spec_files) > 0
spec_files.sort()
@@ -152,22 +209,27 @@ def get_latest_commit(ghidra_repo_dir: PathString) -> str:
)
def update_head(ghidra_repo_dir: PathString, ci: bool) -> bool:
def update_head(
setup_file: Path, spec_file: Path, ghidra_repo_dir: PathString, ci: bool
) -> bool:
"""Update to latest head and make changes to the CMake files"""
tmpdirname = None
if ghidra_repo_dir is None:
tmpdirname = tempfile.TemporaryDirectory()
ghidra_repo_dir = pathlib.Path(tmpdirname.name) / "ghidra"
ghidra_repo_dir = Path(tmpdirname.name) / "ghidra"
clone_ghidra_git(ghidra_repo_dir)
latest_commit = get_latest_commit(ghidra_repo_dir)
did_update_commit = update_head_commit(latest_commit)
did_update_commit = update_head_commit(
setup_file, ghidra_repo_dir, latest_commit, ci
)
if did_update_commit:
if ci:
print(f"::set-output name=short_sha::{latest_commit[:9]}")
print("::set-output name=did_update::true")
update_spec_files(ghidra_repo_dir, HEAD_SPEC_FILE)
update_head_version_file(ghidra_repo_dir)
with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
gh_out.write(f"short_sha={latest_commit[:9]}\n")
gh_out.write("did_update=true\n")
update_spec_files(ghidra_repo_dir, spec_file)
update_head_version_file(setup_file, ghidra_repo_dir)
else:
msg(f"Already at the latest commit: {latest_commit}")
@@ -186,7 +248,7 @@ if __name__ == "__main__":
def dir_path(string):
if string is None:
return string
string = pathlib.Path(string).expanduser().resolve()
string = Path(string).expanduser().resolve()
if string.is_dir():
return string
else:
@@ -207,5 +269,12 @@ if __name__ == "__main__":
)
args = parser.parse_args()
if not update_head(args.ghidra_repo, args.ci):
if args.ci:
assert (
"GITHUB_OUTPUT" in os.environ
), "CI needs `GITHUB_OUTPUT` environment variable set to a file location"
if not update_head(SETUP_GHIDRA_FILE, HEAD_SPEC_FILE, args.ghidra_repo, args.ci):
msg("No update required")
else:
msg("Update required!")