Initial CI to build wheels

This commit is contained in:
Duncan Ogilvie
2026-05-11 19:43:24 +02:00
parent 26763333c2
commit 4c86568eb9
8 changed files with 481 additions and 98 deletions
+151 -85
View File
@@ -2,110 +2,176 @@ name: CI
on:
push:
branches: [master]
pull_request:
branches: [master]
workflow_dispatch:
inputs:
publish:
description: "Publish artifacts to PyPI using trusted publishing"
type: boolean
default: false
# Automatically cancel previous runs of this workflow on the same branch.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
LLVM_VERSION: "21.1.6"
jobs:
test:
sdist:
name: source distribution
# Skip pull requests from this repository; the branch push already builds them.
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository }}
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Setup Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install sdist tools
run: python -m pip install --upgrade build twine
- name: Build sdist
run: python -m build --sdist
- name: Check sdist metadata
run: python -m twine check dist/*.tar.gz
- name: Upload sdist artifact
uses: actions/upload-artifact@v7
with:
name: dist-sdist
path: dist/*.tar.gz
test-wheel:
name: test + wheel (${{ matrix.name }})
# Skip pull requests from this repository; the branch push already builds them.
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository }}
strategy:
fail-fast: false
matrix:
os: [windows-latest, ubuntu-latest, macos-latest]
include:
- name: linux-x86_64
os: ubuntu-24.04
cibw-platform: linux
cibw-archs: x86_64
auditwheel-plat: manylinux_2_28_x86_64
llvm-archive: llvm-21.1.6-linux-x86_64.zip
- name: linux-aarch64
os: ubuntu-24.04-arm
cibw-platform: linux
cibw-archs: aarch64
auditwheel-plat: manylinux_2_28_aarch64
llvm-archive: llvm-21.1.6-linux-aarch64.zip
- name: macos-arm64
os: macos-14
cibw-platform: macos
cibw-archs: arm64
llvm-archive: llvm-21.1.6-macos-arm64.zip
- name: windows-x86_64
os: windows-latest
cibw-platform: windows
cibw-archs: AMD64
llvm-archive: llvm-21.1.6-windows-x86_64.zip
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6
- name: Setup Python 3.12
uses: actions/setup-python@v5
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Setup uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
# ── Install LLVM (Windows) ──────────────────────────────────────
- name: Cache LLVM (Windows)
if: runner.os == 'Windows'
id: cache-llvm-windows
uses: actions/cache@v4
with:
path: llvm-windows
key: llvm-windows-21.1.1-clang-21.1.1
- name: Install LLVM (Windows)
if: runner.os == 'Windows'
shell: bash
run: |
if [ ! -d "llvm-windows/llvm" ]; then
mkdir -p llvm-windows
curl -sL https://github.com/vovkos/llvm-package-windows/releases/download/llvm-21.1.1/llvm-21.1.1-windows-amd64-msvc17-msvcrt.7z -o llvm-windows/llvm.7z
7z x llvm-windows/llvm.7z -ollvm-windows/llvm
curl -sL https://github.com/vovkos/llvm-package-windows/releases/download/clang-21.1.1/clang-21.1.1-windows-amd64-msvc17-msvcrt.7z -o llvm-windows/clang.7z
7z x llvm-windows/clang.7z -ollvm-windows/llvm -aoa
rm -f llvm-windows/llvm.7z llvm-windows/clang.7z
fi
# 7z extracts into a versioned subdirectory — find it
LLVM_PREFIX=$(ls -d llvm-windows/llvm/llvm-*)
# Write .llvm-prefix for cmake and test scripts
echo "$(cygpath -w "$(pwd)/$LLVM_PREFIX")" > .llvm-prefix
# Add bin/ to PATH so DLLs are found at runtime
echo "$(pwd)/$LLVM_PREFIX/bin" >> "$GITHUB_PATH"
# ── Install LLVM (Linux) ────────────────────────────────────────
- name: Install LLVM (Linux)
if: runner.os == 'Linux'
run: |
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo add-apt-repository -y "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-21 main"
sudo apt-get update
sudo apt-get install -y llvm-21-dev llvm-21-tools clang-21 lld-21
echo "/usr/lib/llvm-21" > .llvm-prefix
# ── Install LLVM (macOS) ────────────────────────────────────────
- name: Install LLVM (macOS)
if: runner.os == 'macOS'
run: |
brew install llvm
echo "$(brew --prefix llvm)" > .llvm-prefix
# ── Setup MSVC (Windows) ────────────────────────────────────────
- name: Setup MSVC
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
uses: TheMrMilchmann/setup-msvc-dev@v4.0.0
with:
arch: x64
# ── Build C++ test executables ──────────────────────────────────
- name: Build C++ test executables
- name: Build wheel and run tests
uses: pypa/cibuildwheel@v3.4.1
env:
CIBW_PLATFORM: ${{ matrix.cibw-platform }}
CIBW_ARCHS: ${{ matrix.cibw-archs }}
CIBW_BUILD: "cp312-*"
CIBW_SKIP: "*-musllinux_*"
CIBW_BUILD_VERBOSITY: "1"
CIBW_TEST_REQUIRES: "pytest lit cmake>=3.27 ninja ty"
CIBW_DEPLOYMENT_TARGET_MACOS: "13.0"
CIBW_TEST_COMMAND: >-
python "{project}/tools/ci/test_wheel.py"
# Linux builds run inside manylinux containers. /project is the
# cibuildwheel mount point for this repository inside the container.
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
CIBW_ENVIRONMENT_LINUX: >-
CMAKE_PREFIX_PATH="/project/.llvm"
LD_LIBRARY_PATH="/project/.llvm/lib:$LD_LIBRARY_PATH"
CIBW_BEFORE_BUILD_LINUX: >-
python /project/tools/ci/install_llvm.py
--version "${{ env.LLVM_VERSION }}"
--archive "${{ matrix.llvm-archive }}"
--dest /project/.llvm
CIBW_REPAIR_WHEEL_COMMAND_LINUX: >-
auditwheel repair --plat ${{ matrix.auditwheel-plat }} -w {dest_dir} {wheel}
# macOS builds run directly on the hosted runner. delocate is invoked
# by cibuildwheel to copy dependent dylibs into the wheel.
CIBW_ENVIRONMENT_MACOS: >-
CMAKE_PREFIX_PATH="${{ github.workspace }}/.llvm"
CMAKE_OSX_DEPLOYMENT_TARGET="13.0"
MACOSX_DEPLOYMENT_TARGET="13.0"
DYLD_LIBRARY_PATH="${{ github.workspace }}/.llvm/lib:$DYLD_LIBRARY_PATH"
CIBW_BEFORE_BUILD_MACOS: >-
python "${{ github.workspace }}/tools/ci/install_llvm.py"
--version "${{ env.LLVM_VERSION }}"
--archive "${{ matrix.llvm-archive }}"
--dest "${{ github.workspace }}/.llvm"
# Windows wheels are built on the hosted runner. The CMake install
# step copies LLVM-C.dll next to llvm/__init__.pyd.
CIBW_ENVIRONMENT_WINDOWS: >-
CMAKE_PREFIX_PATH="${{ github.workspace }}\\.llvm"
PATH="${{ github.workspace }}\\.llvm\\bin;$PATH"
CIBW_BEFORE_BUILD_WINDOWS: >-
python "${{ github.workspace }}\\tools\\ci\\install_llvm.py"
--version "${{ env.LLVM_VERSION }}"
--archive "${{ matrix.llvm-archive }}"
--dest "${{ github.workspace }}\\.llvm"
- name: Check wheel metadata
shell: bash
run: |
CMAKE_EXTRA=""
if [ "$RUNNER_OS" == "Windows" ]; then
CMAKE_EXTRA="-DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl"
fi
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release $CMAKE_EXTRA && cmake --build build
run: python -m pip install twine && twine check wheelhouse/*.whl
# ── Build Python package ────────────────────────────────────────
- name: Build Python package
run: uv sync
- name: Upload wheel artifact
uses: actions/upload-artifact@v7
with:
name: dist-${{ matrix.name }}
path: wheelhouse/*.whl
# ── Tests ───────────────────────────────────────────────────────
- name: Golden master tests
run: uv run python run_tests.py
publish:
name: publish to PyPI
needs: [sdist, test-wheel]
runs-on: ubuntu-24.04
if: ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || (github.event_name == 'workflow_dispatch' && inputs.publish) }}
permissions:
id-token: write
contents: read
steps:
- name: Download distributions
uses: actions/download-artifact@v8
with:
pattern: dist-*
merge-multiple: true
path: dist
- name: Regression tests
run: uv run python run_tests.py --regressions
- name: Lit tests (C binary)
if: runner.os != 'Windows'
run: uv run python run_llvm_c_tests.py -v
- name: Lit tests (Python)
if: runner.os != 'Windows'
run: uv run python run_llvm_c_tests.py --use-python -v
- name: Type checking
run: uvx ty check
- name: Publish distributions
uses: pypa/gh-action-pypi-publish@release/v1
+4 -1
View File
@@ -12,6 +12,7 @@ CMakeUserPresets.json
.DS_Store
__pycache__/
dist/
# Test outputs (generated by run_tests.py)
tests/output/
@@ -22,5 +23,7 @@ htmlcov/
.coverage*
coverage.xml
# LLVM prefix path file
# LLVM prefix path/cache files
.llvm-prefix
.llvm/
.llvm-cache/
+10 -2
View File
@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
[project]
name = "llvm-nanobind"
version = "0.0.1"
version = "21.1.6.1"
description = "LLVM-C Python bindings with nanobind."
readme = "README.md"
requires-python = ">=3.12"
@@ -54,7 +54,15 @@ cache-keys = [
]
[tool.ty.src]
exclude = ["llvm-windows"]
exclude = [
".llvm",
".llvm-cache",
".venv",
"build",
"dist",
"htmlcov",
"wheelhouse",
]
[tool.ty.environment]
# Don't include src/ in first-party search path so ty finds the installed
+10 -6
View File
@@ -179,12 +179,16 @@ def build_llvm_c_test_cmd(
if use_python:
# The llvm_c_test package is a dev-only source package, not an installed
# distribution entry point. Make it importable for lit subprocesses while
# preferring the freshly built extension module from build/.
pythonpath_entries = [
str((project_root / "build").resolve()),
str(project_root.resolve()),
]
# distribution entry point. Make it importable for lit subprocesses. Wheel
# tests set LLVM_NANOBIND_TEST_INSTALLED=1 so the installed wheel is tested
# instead of build/llvm.*.
if os.environ.get("LLVM_NANOBIND_TEST_INSTALLED"):
pythonpath_entries = [str(project_root.resolve())]
else:
pythonpath_entries = [
str((project_root / "build").resolve()),
str(project_root.resolve()),
]
if existing_pythonpath := os.environ.get("PYTHONPATH"):
pythonpath_entries.append(existing_pythonpath)
extra_env["PYTHONPATH"] = os.pathsep.join(pythonpath_entries)
+9 -3
View File
@@ -106,9 +106,15 @@ def run_python_test(script: Path) -> tuple[str, str, int]:
# Build command with optional coverage wrapper
cmd = coverage_wrap(script.stem, [str(script)])
# Prefer the freshly built extension while keeping the repository root
# importable for dev-only helper packages such as tools.obfuscation.
pythonpath_entries = [str(BUILD_DIR.resolve()), str(Path(__file__).parent.resolve())]
# Prefer the freshly built extension for normal development runs while
# keeping the repository root importable for dev-only helper packages such
# as tools.obfuscation. Wheel tests set LLVM_NANOBIND_TEST_INSTALLED=1 so
# the installed wheel is tested instead of build/llvm.*.
project_root = Path(__file__).parent.resolve()
if os.environ.get("LLVM_NANOBIND_TEST_INSTALLED"):
pythonpath_entries = [str(project_root)]
else:
pythonpath_entries = [str(BUILD_DIR.resolve()), str(project_root)]
if existing_pythonpath := os.environ.get("PYTHONPATH"):
pythonpath_entries.append(existing_pythonpath)
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""Download and unpack a prebuilt LLVM archive for CI.
The LLVMParty/llvm-builds release assets are ZIP files containing an LLVM
installation root (bin/, include/, lib/, ...). This script downloads one asset,
finds the installation root inside the archive, and installs it at a stable
path used by CMake/scikit-build.
"""
from __future__ import annotations
import argparse
import os
import stat
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from pathlib import Path
from zipfile import ZipFile
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="LLVM version, e.g. 21.1.6")
parser.add_argument(
"--tag",
help="llvm-builds release tag. Defaults to v<version>.",
)
parser.add_argument(
"--repo",
default="LLVMParty/llvm-builds",
help="GitHub repository containing release assets.",
)
parser.add_argument(
"--archive",
required=True,
help="Release asset name, e.g. llvm-21.1.6-linux-x86_64.zip",
)
parser.add_argument(
"--dest",
required=True,
type=Path,
help="Destination LLVM root directory.",
)
return parser.parse_args()
def download(url: str, dest: Path) -> None:
print(f"Downloading {url}", flush=True)
with urllib.request.urlopen(url) as response, dest.open("wb") as out:
shutil.copyfileobj(response, out)
def extract_zip(archive: Path, dest: Path) -> None:
"""Extract a ZIP archive, preserving Unix symlinks when present."""
dest.mkdir(parents=True, exist_ok=True)
# On Unix, prefer unzip because it preserves symlinks and permissions in
# LLVM archives. Fall back to a Python extractor that handles symlinks too.
if os.name != "nt" and shutil.which("unzip"):
subprocess.run(["unzip", "-q", str(archive), "-d", str(dest)], check=True)
return
with ZipFile(archive) as zf:
for info in zf.infolist():
target = dest / info.filename
mode = info.external_attr >> 16
if info.is_dir():
target.mkdir(parents=True, exist_ok=True)
continue
target.parent.mkdir(parents=True, exist_ok=True)
if os.name != "nt" and stat.S_ISLNK(mode):
link_target = zf.read(info).decode()
if target.exists() or target.is_symlink():
target.unlink()
os.symlink(link_target, target)
continue
with zf.open(info) as src, target.open("wb") as out:
shutil.copyfileobj(src, out)
if os.name != "nt" and mode:
target.chmod(mode & 0o777)
def looks_like_llvm_root(path: Path) -> bool:
exe = "llvm-config.exe" if os.name == "nt" else "llvm-config"
return (
(path / "bin" / exe).exists()
and (path / "include" / "llvm-c").is_dir()
and (path / "lib").is_dir()
)
def find_llvm_root(extracted: Path) -> Path:
if looks_like_llvm_root(extracted):
return extracted
candidates = [p for p in extracted.rglob("*") if p.is_dir() and looks_like_llvm_root(p)]
if not candidates:
raise RuntimeError(f"Could not find LLVM root in extracted archive: {extracted}")
# Prefer the shallowest path if the archive contains nested directories.
return sorted(candidates, key=lambda p: (len(p.parts), str(p)))[0]
def install_tree(src: Path, dest: Path) -> None:
if dest.exists():
shutil.rmtree(dest)
dest.parent.mkdir(parents=True, exist_ok=True)
# copytree with symlinks=True preserves Unix dylib/so symlink structure.
shutil.copytree(src, dest, symlinks=True)
def main() -> int:
args = parse_args()
tag = args.tag or f"v{args.version}"
url = f"https://github.com/{args.repo}/releases/download/{tag}/{args.archive}"
with tempfile.TemporaryDirectory(prefix="llvm-download-") as tmp:
tmp_path = Path(tmp)
archive_path = tmp_path / args.archive
extract_dir = tmp_path / "extract"
download(url, archive_path)
extract_zip(archive_path, extract_dir)
llvm_root = find_llvm_root(extract_dir)
install_tree(llvm_root, args.dest)
print(f"Installed LLVM to {args.dest}", flush=True)
llvm_config = args.dest / "bin" / ("llvm-config.exe" if os.name == "nt" else "llvm-config")
if llvm_config.exists():
version = subprocess.check_output([str(llvm_config), "--version"], text=True).strip()
print(f"LLVM version: {version}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Run the repository test suites against an installed wheel.
cibuildwheel installs the freshly built wheel into an isolated test
environment before running this script. The script then builds the C++ test
executables from source, but keeps Python imports pointed at the installed
wheel rather than build/llvm.*.
"""
from __future__ import annotations
import importlib.metadata
import os
import shutil
import subprocess
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
BUILD_DIR = PROJECT_ROOT / "build"
def run(args: list[str], *, env: dict[str, str] | None = None) -> None:
print("+", " ".join(args), flush=True)
subprocess.run(args, cwd=PROJECT_ROOT, env=env, check=True)
def find_llvm_root() -> Path:
candidates: list[Path] = []
if cmake_prefix_path := os.environ.get("CMAKE_PREFIX_PATH"):
candidates.extend(Path(p) for p in cmake_prefix_path.split(os.pathsep) if p)
if llvm_root := os.environ.get("LLVM_ROOT"):
candidates.append(Path(llvm_root))
prefix_file = PROJECT_ROOT / ".llvm-prefix"
if prefix_file.exists():
candidates.append(Path(prefix_file.read_text(encoding="utf-8").strip()))
candidates.append(PROJECT_ROOT / ".llvm")
for candidate in candidates:
llvm_config = candidate / "bin" / ("llvm-config.exe" if sys.platform == "win32" else "llvm-config")
if llvm_config.exists():
return candidate.resolve()
raise RuntimeError(
"Could not find LLVM root. Set CMAKE_PREFIX_PATH or LLVM_ROOT to the prebuilt LLVM prefix."
)
def make_test_env(llvm_root: Path) -> dict[str, str]:
env = os.environ.copy()
env["LLVM_NANOBIND_TEST_INSTALLED"] = "1"
env["CMAKE_PREFIX_PATH"] = str(llvm_root)
env["LLVM_ROOT"] = str(llvm_root)
path_entries = [str(llvm_root / "bin")]
if existing_path := env.get("PATH"):
path_entries.append(existing_path)
env["PATH"] = os.pathsep.join(path_entries)
if sys.platform == "darwin":
key = "DYLD_LIBRARY_PATH"
env[key] = os.pathsep.join([str(llvm_root / "lib"), env.get(key, "")]).rstrip(os.pathsep)
elif sys.platform != "win32":
key = "LD_LIBRARY_PATH"
env[key] = os.pathsep.join([str(llvm_root / "lib"), env.get(key, "")]).rstrip(os.pathsep)
return env
def configure_and_build_cpp_tests(env: dict[str, str]) -> None:
if BUILD_DIR.exists():
shutil.rmtree(BUILD_DIR)
cmake_args = [
"cmake",
"-B",
str(BUILD_DIR),
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=Release",
]
if sys.platform == "win32":
cmake_args.extend([
"-DCMAKE_C_COMPILER=clang-cl",
"-DCMAKE_CXX_COMPILER=clang-cl",
])
run(cmake_args, env=env)
run(["cmake", "--build", str(BUILD_DIR)], env=env)
def run_extra_standalone_tests(env: dict[str, str]) -> None:
scripts = [
"tests/test_binary.py",
"tests/test_bitcode_linker.py",
"tests/test_feature_matrix.py",
"tests/test_function_extended.py",
"tests/test_passbuilder.py",
"tests/test_target_codegen.py",
]
for script in scripts:
run([sys.executable, script], env=env)
def run_extra_cpp_tests(env: dict[str, str]) -> None:
exe_suffix = ".exe" if sys.platform == "win32" else ""
tests = [
"test_target_codegen",
"test_bitcode_linker",
"test_passbuilder",
"test_function_extended",
"test_symbol_size_crash",
]
for test in tests:
run([str(BUILD_DIR / f"{test}{exe_suffix}")], env=env)
def main() -> int:
llvm_root = find_llvm_root()
(PROJECT_ROOT / ".llvm-prefix").write_text(str(llvm_root) + "\n", encoding="utf-8")
env = make_test_env(llvm_root)
import llvm
print(f"Testing installed llvm-nanobind {importlib.metadata.version('llvm-nanobind')}")
print(f"llvm module: {llvm.__file__}")
print(f"LLVM root: {llvm_root}")
configure_and_build_cpp_tests(env)
run([sys.executable, "run_tests.py"], env=env)
run([sys.executable, "run_tests.py", "--regressions"], env=env)
run([sys.executable, "run_llvm_c_tests.py", "-v"], env=env)
run([sys.executable, "run_llvm_c_tests.py", "--use-python", "-v"], env=env)
run_extra_standalone_tests(env)
run_extra_cpp_tests(env)
run([sys.executable, "-m", "pytest"], env=env)
run([shutil.which("ty") or "ty", "check"], env=env)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Generated
+1 -1
View File
@@ -114,7 +114,7 @@ wheels = [
[[package]]
name = "llvm-nanobind"
version = "0.0.1"
version = "21.1.6.1"
source = { editable = "." }
[package.dev-dependencies]