From 3c38995c6eb731c2d851212eef44640aba1edefd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 5 Jun 2026 15:00:41 -0400 Subject: [PATCH] Add code coverage + HTML report machinery This adds a new `cargo dev coverage` subcommand, which invokes nextest under our new coverage pesudo-profile. This profile in turn enables `-C instrument-coverage`. `cargo dev coverage` also takes care of merging raw profiling samples into profdata and LCOV reports for us, which our new `coverage-html-report.py` script can turn into a readable (but very large) HTML report. The basic flow for this is: ``` $ # any nextest flags will work $ cargo dev coverage ... ``` That will output something like: ``` Merged profile: .../astral-sh/uv/target/coverage/profdata/b46cc0f3-7cd0-4309-9df2-10ab653e6dd8.profdata LCOV profile: .../astral-sh/uv/target/coverage/lcov/b46cc0f3-7cd0-4309-9df2-10ab653e6dd8.lcov Generate an HTML report with: uv run --script scripts/coverage-html-report.py b46cc0f3-7cd0-4309-9df2-10ab653e6dd8 ``` And then you can run: ``` uv run --script scripts/coverage-html-report.py b46cc0f3-7cd0-4309-9df2-10ab653e6dd8 ``` (or add `--open` to auto-open in the browser). Under the hood, this only uses existing dependencies plus Rust's own LLVM toolchain, plus coverage.py for the HTML report generation. Signed-off-by: William Woodruff MVP coverage workflow Signed-off-by: William Woodruff MVP coverage workflow, take 2 Signed-off-by: William Woodruff Use a pool of 16 raw profiles for pooling Signed-off-by: William Woodruff Avoid some Windows UNC pain Signed-off-by: William Woodruff Fix rustflags for coverage on Windows Signed-off-by: William Woodruff Remove coverage ID chum Signed-off-by: William Woodruff Avoid Windows nonsense Signed-off-by: William Woodruff Add coverage guide to CONTRIBUTING Signed-off-by: William Woodruff Be less strict about the ID Signed-off-by: William Woodruff Prettier Signed-off-by: William Woodruff --- .cargo/coverage.toml | 8 + .github/workflows/coverage.yml | 18 ++ .github/workflows/test.yml | 185 +++++++++++++++-- CONTRIBUTING.md | 55 ++++++ Cargo.lock | 1 + crates/uv-dev/Cargo.toml | 1 + crates/uv-dev/src/coverage.rs | 341 ++++++++++++++++++++++++++++++++ crates/uv-dev/src/lib.rs | 5 + scripts/coverage-html-report.py | 180 +++++++++++++++++ 9 files changed, 775 insertions(+), 19 deletions(-) create mode 100644 .cargo/coverage.toml create mode 100644 .github/workflows/coverage.yml create mode 100644 crates/uv-dev/src/coverage.rs create mode 100644 scripts/coverage-html-report.py diff --git a/.cargo/coverage.toml b/.cargo/coverage.toml new file mode 100644 index 0000000000..617204561c --- /dev/null +++ b/.cargo/coverage.toml @@ -0,0 +1,8 @@ +[build] +rustflags = ["-C", "instrument-coverage"] +target-dir = "target/coverage" + +# Target-specific rustflags take precedence over build.rustflags. Repeat the +# instrumentation flag so it is combined with Windows' static CRT flag. +[target.'cfg(all(target_env = "msvc", target_os = "windows"))'] +rustflags = ["-C", "instrument-coverage"] diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..43f9836529 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,18 @@ +name: Coverage + +on: + pull_request: + types: [labeled] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + test: + if: ${{ github.event.label.name == 'coverage' }} + uses: ./.github/workflows/test.yml + with: + save-rust-cache: "false" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af8ba9b669..f98700c83b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,7 @@ env: CARGO_TERM_COLOR: always PYTHON_VERSION: "3.12" RUSTUP_MAX_RETRIES: 10 + COLLECT_COVERAGE: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'coverage') }} jobs: # We use the large GitHub actions runners @@ -25,7 +26,7 @@ jobs: # See: https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-ubuntu-and-windows-larger-runners cargo-test-linux: - timeout-minutes: 10 + timeout-minutes: ${{ (github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'coverage')) && 20 || 10 }} runs-on: depot-ubuntu-24.04-16 name: "cargo test on linux" steps: @@ -47,6 +48,10 @@ jobs: with: version: "0.11.21" + - name: "Install LLVM tools" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + run: rustup component add llvm-tools + - name: "Install required Python versions" run: uv python install @@ -90,6 +95,7 @@ jobs: tool: cargo-nextest - name: "Cargo test" + id: test env: # Retry more than default to reduce flakes in CI UV_HTTP_RETRIES: 5 @@ -102,11 +108,53 @@ jobs: INSTA_UPDATE: new INSTA_PENDING_DIR: ${{ github.workspace }}/pending-snapshots run: | - cargo nextest run \ - --cargo-profile fast-build \ - --features test-python-patch,native-auth,secret-service \ - --workspace \ + test_args=( + --cargo-profile fast-build + --features test-python-patch,native-auth,secret-service + --workspace --profile ci-linux + ) + if [[ "$COLLECT_COVERAGE" == "true" ]]; then + tracking_id="${GITHUB_SHA:0:16}" + cargo dev coverage --id "$tracking_id" -- "${test_args[@]}" + echo "tracking_id=$tracking_id" >> "$GITHUB_OUTPUT" + else + cargo nextest run "${test_args[@]}" + fi + + - name: "Generate HTML coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + env: + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + uv run --script scripts/coverage-html-report.py "$COVERAGE_ID" + test -f "target/coverage/lcov/$COVERAGE_ID.lcov" + test -f "target/coverage/html/$COVERAGE_ID/index.html" + + - name: "Upload coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + id: upload-coverage + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-linux-${{ github.sha }} + path: | + target/coverage/html/${{ steps.test.outputs.tracking_id }}/ + target/coverage/lcov/${{ steps.test.outputs.tracking_id }}.lcov + if-no-files-found: error + retention-days: 14 + + - name: "Summarize coverage" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + env: + ARTIFACT_URL: ${{ steps.upload-coverage.outputs.artifact-url }} + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + { + echo "## Linux coverage" + echo + echo "- Tracking ID: \`$COVERAGE_ID\`" + echo "- Artifact: [$ARTIFACT_URL]($ARTIFACT_URL)" + } >> "$GITHUB_STEP_SUMMARY" - name: "Upload pending snapshots" if: ${{ failure() }} @@ -128,9 +176,9 @@ jobs: retention-days: 14 cargo-test-macos: - timeout-minutes: 20 + timeout-minutes: ${{ (github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'coverage')) && 30 || 20 }} # Only run macOS tests on main without opt-in - if: ${{ inputs.test-macos == 'true' }} + if: ${{ inputs.test-macos == 'true' || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'coverage') }} runs-on: depot-macos-15 name: "cargo test on macos" steps: @@ -145,6 +193,10 @@ jobs: - name: "Install Rust toolchain" run: rustup show + - name: "Install LLVM tools" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + run: rustup component add llvm-tools + - name: "Create HFS+ disk image (no reflink support)" run: | hdiutil create -size 256m -fs HFS+ -volname NoReflink /tmp/noreflink.dmg @@ -164,6 +216,7 @@ jobs: tool: cargo-nextest - name: "Cargo test" + id: test env: # Retry more than default to reduce flakes in CI UV_HTTP_RETRIES: 5 @@ -177,12 +230,54 @@ jobs: INSTA_UPDATE: new INSTA_PENDING_DIR: ${{ github.workspace }}/pending-snapshots run: | - cargo nextest run \ - --cargo-profile fast-build \ - --no-default-features \ - --features test-python,test-python-managed,test-pypi,test-git,test-git-lfs,performance,test-crates-io,native-auth,apple-native \ - --workspace \ + test_args=( + --cargo-profile fast-build + --no-default-features + --features test-python,test-python-managed,test-pypi,test-git,test-git-lfs,performance,test-crates-io,native-auth,apple-native + --workspace --profile ci-macos + ) + if [[ "$COLLECT_COVERAGE" == "true" ]]; then + tracking_id="${GITHUB_SHA:0:16}" + cargo dev coverage --id "$tracking_id" -- "${test_args[@]}" + echo "tracking_id=$tracking_id" >> "$GITHUB_OUTPUT" + else + cargo nextest run "${test_args[@]}" + fi + + - name: "Generate HTML coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + env: + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + uv run --script scripts/coverage-html-report.py "$COVERAGE_ID" + test -f "target/coverage/lcov/$COVERAGE_ID.lcov" + test -f "target/coverage/html/$COVERAGE_ID/index.html" + + - name: "Upload coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + id: upload-coverage + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-macos-${{ github.sha }} + path: | + target/coverage/html/${{ steps.test.outputs.tracking_id }}/ + target/coverage/lcov/${{ steps.test.outputs.tracking_id }}.lcov + if-no-files-found: error + retention-days: 14 + + - name: "Summarize coverage" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + env: + ARTIFACT_URL: ${{ steps.upload-coverage.outputs.artifact-url }} + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + { + echo "## macOS coverage" + echo + echo "- Tracking ID: \`$COVERAGE_ID\`" + echo "- Artifact: [$ARTIFACT_URL]($ARTIFACT_URL)" + } >> "$GITHUB_STEP_SUMMARY" - name: "Upload pending snapshots" if: ${{ failure() }} @@ -204,7 +299,7 @@ jobs: retention-days: 14 cargo-test-windows: - timeout-minutes: 15 + timeout-minutes: ${{ (github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'coverage')) && 30 || 15 }} runs-on: namespace-profile-windows-2022-x86-64-16 name: "cargo test on windows ${{ matrix.partition }} of 3" strategy: @@ -242,6 +337,11 @@ jobs: working-directory: ${{ env.UV_WORKSPACE }} run: rustup show + - name: "Install LLVM tools" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + working-directory: ${{ env.UV_WORKSPACE }} + run: rustup component add llvm-tools + - name: "Create NTFS test directory (low hardlink limit)" run: New-Item -Path "C:\uv" -ItemType Directory -Force @@ -256,6 +356,7 @@ jobs: tool: cargo-nextest - name: "Cargo test" + id: test working-directory: ${{ env.UV_WORKSPACE }} env: # Retry more than default to reduce flakes in CI @@ -272,13 +373,59 @@ jobs: INSTA_PENDING_DIR: ${{ github.workspace }}/pending-snapshots shell: bash run: | - cargo nextest run \ - --cargo-profile fast-build \ - --no-default-features \ - --features test-python,test-pypi,test-python-managed,test-windows-registry,native-auth,windows-native \ - --workspace \ - --profile ci-windows \ + test_args=( + --cargo-profile fast-build + --no-default-features + --features test-python,test-pypi,test-python-managed,test-windows-registry,native-auth,windows-native + --workspace + --profile ci-windows --partition hash:${{ matrix.partition }}/3 + ) + if [[ "$COLLECT_COVERAGE" == "true" ]]; then + tracking_id="${GITHUB_SHA:0:16}" + cargo dev coverage --id "$tracking_id" -- "${test_args[@]}" + echo "tracking_id=$tracking_id" >> "$GITHUB_OUTPUT" + else + cargo nextest run "${test_args[@]}" + fi + + - name: "Generate HTML coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + working-directory: ${{ env.UV_WORKSPACE }} + shell: bash + env: + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + uv run --script scripts/coverage-html-report.py "$COVERAGE_ID" + test -f "target/coverage/lcov/$COVERAGE_ID.lcov" + test -f "target/coverage/html/$COVERAGE_ID/index.html" + + - name: "Upload coverage report" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + id: upload-coverage + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-windows-${{ matrix.partition }}-${{ github.sha }} + path: | + ${{ env.UV_WORKSPACE }}/target/coverage/html/${{ steps.test.outputs.tracking_id }}/ + ${{ env.UV_WORKSPACE }}/target/coverage/lcov/${{ steps.test.outputs.tracking_id }}.lcov + if-no-files-found: error + retention-days: 14 + + - name: "Summarize coverage" + if: ${{ env.COLLECT_COVERAGE == 'true' }} + working-directory: ${{ env.UV_WORKSPACE }} + shell: bash + env: + ARTIFACT_URL: ${{ steps.upload-coverage.outputs.artifact-url }} + COVERAGE_ID: ${{ steps.test.outputs.tracking_id }} + run: | + { + echo "## Windows coverage partition ${{ matrix.partition }}" + echo + echo "- Tracking ID: \`$COVERAGE_ID\`" + echo "- Artifact: [$ARTIFACT_URL]($ARTIFACT_URL)" + } >> "$GITHUB_STEP_SUMMARY" - name: "Upload pending snapshots" if: ${{ failure() }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afb54f962c..b465062de5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -247,6 +247,61 @@ uv run resolver \ ../test/requirements/trio.in ``` +## Code coverage + +You can run uv's unit and integration tests under instrumentation to get a sense for which APIs +currently lack test coverage. + +To performan a coverage-instrumented run, you'll need some additional LLVM tools. We strongly +recommend adding these via `rustup` so that they match your current Rust toolchain's LLVM version: + +```shell +rustup component add llvm-tools +``` + +From there, you can run `cargo dev coverage` to run the tests with coverage. `cargo dev coverage` +runs `nextest` under the hood, so you can limit a coverage run like you would for normal test runs. + +Examples: + +```shell +# run all tests under coverage +cargo dev coverage + +# run just the uv-audit member's tests +cargo dev coverage -- -p uv-audit + +# optional: pass an explicit tracking ID instead of generating one +cargo dev coverage --id demo +``` + +By default, `cargo dev coverage` will generate a tracking ID and print it; you'll see something like +this: + +```console +Coverage tracking ID: WuSYqZVUl0R9YPka +Raw profiles: /Users/ww/oss/astral-sh/uv/target/coverage/profraw/WuSYqZVUl0R9YPka + +[... normal test output ...] +[... normal test output ...] +[... normal test output ...] + +Merged profile: /Users/ww/oss/astral-sh/uv/target/coverage/profdata/WuSYqZVUl0R9YPka.profdata +LCOV profile: /Users/ww/oss/astral-sh/uv/target/coverage/lcov/WuSYqZVUl0R9YPka.lcov +Generate an HTML report with: uv run --script scripts/coverage-html-report.py WuSYqZVUl0R9YPka +``` + +Then, to turn your run's coverage into a human-readable HTML report: + +```shell +uv run --script scripts/coverage-html-report.py TRACKING_ID + +# optional: open the HTML report in the default browser +uv run --script scripts/coverage-html-report.py --open TRACKING_ID +``` + +Note that HTML reports can be very large, and may take some time to render in your browser. + ### Analyzing concurrency You can use [tracing-durations-export](https://github.com/konstin/tracing-durations-export) to diff --git a/Cargo.lock b/Cargo.lock index 6220deab35..9d42808ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6253,6 +6253,7 @@ dependencies = [ "uv-distribution-types", "uv-errors", "uv-extract", + "uv-fastid", "uv-git", "uv-installer", "uv-macros", diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index 7ce657b8c5..a71815753a 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -63,6 +63,7 @@ tokio-util = { workspace = true } tracing = { workspace = true } tracing-durations-export = { workspace = true, features = ["plot"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } +uv-fastid = { workspace = true } uv-performance-memory-allocator = { path = "../uv-performance-memory-allocator", optional = true } walkdir = { workspace = true } diff --git a/crates/uv-dev/src/coverage.rs b/crates/uv-dev/src/coverage.rs new file mode 100644 index 0000000000..ad74f6bd3a --- /dev/null +++ b/crates/uv-dev/src/coverage.rs @@ -0,0 +1,341 @@ +use std::env; +use std::ffi::OsString; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anstream::println; +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde_json::Value; +use uv_fastid::Id; + +use crate::ROOT_DIR; + +const CARGO_MESSAGE_REASONS: [&str; 4] = [ + "compiler-artifact", + "compiler-message", + "build-script-executed", + "build-finished", +]; + +fn parse_coverage_id(value: &str) -> Result { + if value.is_empty() + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return Err(format!( + "invalid ID: {value:?} (must be non-empty and use only [A-Za-z0-9-_])" + )); + } + Ok(value.to_string()) +} + +#[derive(Parser)] +pub(crate) struct CoverageArgs { + /// The ID to use for the coverage run. + #[arg(long, value_parser = parse_coverage_id)] + id: Option, + + /// Additional arguments to pass to `cargo nextest run`. + #[arg(last = true)] + nextest_args: Vec, +} + +pub(crate) fn coverage(args: CoverageArgs) -> Result<()> { + let root = fs_err::canonicalize(ROOT_DIR).context("failed to locate the workspace root")?; + let llvm_tools = find_llvm_tools(&root)?; + + let tracking_id = args.id.unwrap_or_else(|| Id::insecure().to_string()); + let raw_profiles = root + .join("target") + .join("coverage") + .join("profraw") + .join(&tracking_id); + let merged_profiles = root.join("target").join("coverage").join("profdata"); + let merged_profile = merged_profiles.join(format!("{tracking_id}.profdata")); + let lcov_profiles = root.join("target").join("coverage").join("lcov"); + let lcov_profile = lcov_profiles.join(format!("{tracking_id}.lcov")); + + fs_err::create_dir_all(&raw_profiles) + .with_context(|| format!("failed to create `{}`", raw_profiles.display()))?; + + println!("Coverage tracking ID: {tracking_id}"); + println!("Raw profiles: {}", raw_profiles.display()); + + // Bound disk usage while retaining enough merge slots for concurrent test processes. + let profile_pattern = raw_profiles.join("%16m.profraw"); + let mut child = Command::new("cargo") + .args([ + "nextest", + "run", + "--config", + ".cargo/coverage.toml", + "--cargo-message-format", + "json-render-diagnostics", + ]) + .args(args.nextest_args) + .current_dir(&root) + .env("LLVM_PROFILE_FILE", profile_pattern) + .stdout(Stdio::piped()) + .spawn() + .context("failed to run `cargo nextest run`")?; + + let Some(stdout) = child.stdout.take() else { + bail!("failed to read output from `cargo nextest run`"); + }; + let binaries = collect_coverage_binaries(BufReader::new(stdout))?; + let status = child + .wait() + .context("failed to wait for `cargo nextest run`")?; + if !status.success() { + bail!("`cargo nextest run` failed with {status}"); + } + + let profraw_files = collect_profraw_files(&raw_profiles)?; + + if profraw_files.is_empty() { + bail!( + "`cargo nextest run` produced no raw coverage profiles in `{}`", + raw_profiles.display() + ); + } + + fs_err::create_dir_all(&merged_profiles) + .with_context(|| format!("failed to create `{}`", merged_profiles.display()))?; + + // Use an input file to avoid exceeding the Windows command-line length limit when there are + // many raw profiles. + let mut profraw_file_list = tempfile::NamedTempFile::new_in(&merged_profiles) + .context("failed to create temporary raw profile list")?; + for profraw_file in &profraw_files { + writeln!(profraw_file_list, "{}", profraw_file.display())?; + } + profraw_file_list + .flush() + .context("failed to flush temporary raw profile list")?; + + let input_files_argument = format!("--input-files={}", profraw_file_list.path().display()); + let status = Command::new(&llvm_tools.profdata) + .args(["merge", "-sparse"]) + .arg(input_files_argument) + .arg("-o") + .arg(&merged_profile) + .status() + .with_context(|| format!("failed to run `{}`", llvm_tools.profdata.display()))?; + + if !status.success() { + bail!("`llvm-profdata merge` failed with {status}"); + } + + println!("Merged profile: {}", merged_profile.display()); + + if binaries.is_empty() { + bail!("`cargo nextest run` produced no executable artifacts for coverage export"); + } + + fs_err::create_dir_all(&lcov_profiles) + .with_context(|| format!("failed to create `{}`", lcov_profiles.display()))?; + export_lcov( + &llvm_tools.cov, + &merged_profile, + &binaries, + &root, + &lcov_profile, + )?; + + println!("LCOV profile: {}", lcov_profile.display()); + println!( + "Generate an HTML report with: uv run --script scripts/coverage-html-report.py {tracking_id}" + ); + + Ok(()) +} + +fn collect_coverage_binaries(reader: impl BufRead) -> Result> { + let mut binaries = Vec::new(); + let mut stdout = std::io::stdout().lock(); + + for line in reader.lines() { + let line = line.context("failed to read output from `cargo nextest run`")?; + let Ok(message) = serde_json::from_str::(&line) else { + writeln!(stdout, "{line}")?; + stdout.flush()?; + continue; + }; + let Some(reason) = message.get("reason").and_then(Value::as_str) else { + writeln!(stdout, "{line}")?; + stdout.flush()?; + continue; + }; + + if reason == "compiler-artifact" + && let Some(executable) = message.get("executable").and_then(Value::as_str) + { + let is_test = message + .pointer("/profile/test") + .and_then(Value::as_bool) + .unwrap_or(false); + let is_binary = message + .pointer("/target/kind") + .and_then(Value::as_array) + .is_some_and(|kinds| kinds.iter().any(|kind| kind.as_str() == Some("bin"))); + if is_test || is_binary { + binaries.push(PathBuf::from(executable)); + } + } + + if !CARGO_MESSAGE_REASONS.contains(&reason) { + writeln!(stdout, "{line}")?; + stdout.flush()?; + } + } + + binaries.sort_unstable(); + binaries.dedup(); + Ok(binaries) +} + +fn collect_profraw_files(raw_profiles: &Path) -> Result> { + let mut profraw_files = Vec::new(); + for entry in fs_err::read_dir(raw_profiles) + .with_context(|| format!("failed to read `{}`", raw_profiles.display()))? + { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_file() + && path + .extension() + .is_some_and(|extension| extension == "profraw") + { + profraw_files.push(path); + } + } + profraw_files.sort_unstable(); + Ok(profraw_files) +} + +fn export_lcov( + llvm_cov: &Path, + merged_profile: &Path, + binaries: &[PathBuf], + root: &Path, + lcov_profile: &Path, +) -> Result<()> { + let Some((binary, additional_binaries)) = binaries.split_first() else { + bail!("no executable artifacts were provided for coverage export"); + }; + + let mut command = Command::new(llvm_cov); + command + .arg("export") + .arg(binary) + .arg(format!("--instr-profile={}", merged_profile.display())) + .args([ + r"--ignore-filename-regex=[/\\]\.cargo[/\\]", + r"--ignore-filename-regex=[/\\]rustc[/\\]", + r"--ignore-filename-regex=[/\\]\.rustup[/\\]toolchains[/\\]", + r"--ignore-filename-regex=[/\\]target[/\\]", + "--format=lcov", + ]) + .stdout(Stdio::piped()); + for binary in additional_binaries { + command.arg("-object").arg(binary); + } + + let mut child = command + .spawn() + .with_context(|| format!("failed to run `{}`", llvm_cov.display()))?; + let Some(stdout) = child.stdout.take() else { + bail!("failed to read output from `llvm-cov export`"); + }; + + let mut temp_file = tempfile::NamedTempFile::new_in( + lcov_profile + .parent() + .context("LCOV output path must have a parent")?, + ) + .context("failed to create temporary LCOV output")?; + for line in BufReader::new(stdout).lines() { + let line = line.context("failed to read output from `llvm-cov export`")?; + if let Some(source) = line.strip_prefix("SF:") { + let source = Path::new(source); + if let Ok(relative) = source.strip_prefix(root) { + writeln!( + temp_file, + "SF:{}", + relative.to_string_lossy().replace('\\', "/") + )?; + } else { + writeln!( + temp_file, + "SF:{}", + source.to_string_lossy().replace('\\', "/") + )?; + } + } else { + writeln!(temp_file, "{line}")?; + } + } + + let status = child + .wait() + .context("failed to wait for `llvm-cov export`")?; + if !status.success() { + bail!("`llvm-cov export` failed with {status}"); + } + + temp_file.flush().context("failed to flush LCOV output")?; + temp_file + .persist(lcov_profile) + .map_err(|error| error.error) + .with_context(|| format!("failed to write `{}`", lcov_profile.display()))?; + + Ok(()) +} + +struct LlvmTools { + profdata: PathBuf, + cov: PathBuf, +} + +fn find_llvm_tools(root: &Path) -> Result { + let output = Command::new("rustc") + .args(["--print", "target-libdir"]) + .current_dir(root) + .output() + .context("failed to run `rustc --print=target-libdir`")?; + + if !output.status.success() { + bail!( + "`rustc --print=target-libdir` failed with {}", + output.status + ); + } + + let target_libdir = String::from_utf8(output.stdout) + .context("`rustc --print=target-libdir` returned a non-UTF-8 path")?; + let target_libdir = Path::new(target_libdir.trim()); + let Some(target_dir) = target_libdir.parent() else { + bail!( + "`rustc --print=target-libdir` returned an invalid path: `{}`", + target_libdir.display() + ); + }; + + let tool_dir = target_dir.join("bin"); + let profdata = tool_dir.join(format!("llvm-profdata{}", env::consts::EXE_SUFFIX)); + let cov = tool_dir.join(format!("llvm-cov{}", env::consts::EXE_SUFFIX)); + for tool in [&profdata, &cov] { + if !tool.is_file() { + bail!( + "`{}` was not found at `{}`; install it with `rustup component add llvm-tools`", + tool.file_name().unwrap_or_default().to_string_lossy(), + tool.display() + ); + } + } + + Ok(LlvmTools { profdata, cov }) +} diff --git a/crates/uv-dev/src/lib.rs b/crates/uv-dev/src/lib.rs index 293a8a8f15..74264b8d0b 100644 --- a/crates/uv-dev/src/lib.rs +++ b/crates/uv-dev/src/lib.rs @@ -8,6 +8,7 @@ use uv_settings::EnvironmentOptions; use crate::clear_compile::ClearCompileArgs; use crate::compile::CompileArgs; +use crate::coverage::CoverageArgs; use crate::generate_all::Args as GenerateAllArgs; use crate::generate_cli_reference::Args as GenerateCliReferenceArgs; use crate::generate_env_vars_reference::Args as GenerateEnvVarsReferenceArgs; @@ -23,6 +24,7 @@ use crate::wheel_metadata::WheelMetadataArgs; mod clear_compile; mod compile; +mod coverage; mod generate_all; mod generate_cli_reference; mod generate_env_vars_reference; @@ -47,6 +49,8 @@ enum Cli { Compile(CompileArgs), /// Remove all `.pyc` in the tree. ClearCompile(ClearCompileArgs), + /// Run tests and collect LLVM coverage profiles. + Coverage(CoverageArgs), /// List all packages from a Simple API index. ListPackages(ListPackagesArgs), /// Run all code and documentation generation steps. @@ -80,6 +84,7 @@ pub async fn run() -> Result<()> { Cli::ValidateZip(args) => validate_zip::validate_zip(args, environment).await?, Cli::Compile(args) => compile::compile(args).await?, Cli::ClearCompile(args) => clear_compile::clear_compile(&args)?, + Cli::Coverage(args) => coverage::coverage(args)?, Cli::ListPackages(args) => list_packages::list_packages(args, environment).await?, Cli::GenerateAll(args) => generate_all::main(&args).await?, Cli::GenerateJSONSchema(args) => generate_json_schema::main(&args)?, diff --git a/scripts/coverage-html-report.py b/scripts/coverage-html-report.py new file mode 100644 index 0000000000..db5da8ce39 --- /dev/null +++ b/scripts/coverage-html-report.py @@ -0,0 +1,180 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["coverage"] +# /// + +# Adapted from pyca/cryptography's merge_rust_coverage.py +# See: +# License: + +from __future__ import annotations + +import argparse +import collections +import collections.abc +import pathlib +import webbrowser + +import coverage + +CoverageData = collections.abc.Mapping[str, collections.abc.Mapping[int, int]] +ID_ALPHABET = frozenset( + "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) +# This report uses only source and line records; ignore function, branch, and summary metadata. +IGNORED_RECORDS = { + "BRDA", + "BRF", + "BRH", + "FN", + "FNDA", + "FNF", + "FNH", + "LF", + "LH", + "TN", +} +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent + + +class RustCoveragePlugin(coverage.CoveragePlugin): + def __init__(self, coverage_data: CoverageData) -> None: + super().__init__() + self._data = coverage_data + + def file_reporter(self, filename: str) -> coverage.FileReporter: + return RustCoverageFileReporter(filename, self._data[filename]) + + +class RustCoverageFileReporter(coverage.FileReporter): + def __init__( + self, filename: str, coverage_data: collections.abc.Mapping[int, int] + ) -> None: + super().__init__(filename) + self._data = coverage_data + + def lines(self) -> set[int]: + return set(self._data) + + +def tracking_id(value: str) -> str: + if not value or not set(value) <= ID_ALPHABET: + raise argparse.ArgumentTypeError( + "coverage ID must be non-empty and use only valid characters" + ) + return value + + +def parse_lcov(path: pathlib.Path) -> CoverageData: + raw_data: collections.defaultdict[str, collections.defaultdict[int, int]] = ( + collections.defaultdict(lambda: collections.defaultdict(int)) + ) + current_file: str | None = None + + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + line = line.strip() + if not line: + continue + if line == "end_of_record": + if current_file is None: + raise ValueError(f"{path}:{line_number}: unexpected end_of_record") + current_file = None + continue + + prefix, separator, suffix = line.partition(":") + if not separator: + raise ValueError(f"{path}:{line_number}: malformed LCOV record") + + if prefix == "SF": + if current_file is not None: + raise ValueError( + f"{path}:{line_number}: source record was not terminated" + ) + source = pathlib.Path(suffix) + if not source.is_absolute(): + source = REPO_ROOT / source + source = source.resolve() + if not source.is_relative_to(REPO_ROOT): + raise ValueError( + f"{path}:{line_number}: source is outside the repository: {source}" + ) + if not source.is_file(): + raise ValueError( + f"{path}:{line_number}: source file does not exist: {source}" + ) + current_file = str(source) + elif prefix == "DA": + if current_file is None: + raise ValueError( + f"{path}:{line_number}: DA record has no source file" + ) + fields = suffix.split(",") + if len(fields) < 2: + raise ValueError(f"{path}:{line_number}: malformed DA record") + try: + source_line = int(fields[0]) + count = int(fields[1]) + except ValueError as error: + raise ValueError( + f"{path}:{line_number}: malformed DA record" + ) from error + raw_data[current_file][source_line] += count + elif prefix not in IGNORED_RECORDS: + raise ValueError( + f"{path}:{line_number}: unsupported LCOV record: {prefix}" + ) + + if current_file is not None: + raise ValueError(f"{path}: unterminated source record") + if not raw_data: + raise ValueError(f"{path}: no line coverage records found") + + return raw_data + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate terminal and HTML reports from a Rust coverage run." + ) + parser.add_argument("id", type=tracking_id, help="Coverage tracking ID") + parser.add_argument( + "--open", + action="store_true", + help="Open the generated HTML report in the default browser", + ) + args = parser.parse_args() + + lcov_path = REPO_ROOT / "target" / "coverage" / "lcov" / f"{args.id}.lcov" + if not lcov_path.is_file(): + parser.error(f"LCOV file does not exist: {lcov_path}") + + raw_data = parse_lcov(lcov_path) + covered_lines = { + filename: {line for line, count in lines.items() if count > 0} + for filename, lines in raw_data.items() + } + + plugin_name = "None.RustCoveragePlugin" + cov = coverage.Coverage( + data_file=None, + config_file=False, + plugins=[ + lambda registry: registry.add_file_tracer(RustCoveragePlugin(raw_data)) + ], + ) + data = cov.get_data() + data.add_lines(covered_lines) + data.add_file_tracers(dict.fromkeys(raw_data, plugin_name)) + + cov.report(show_missing=True) + html_directory = REPO_ROOT / "target" / "coverage" / "html" / args.id + cov.html_report(directory=str(html_directory)) + html_report = html_directory / "index.html" + print(f"HTML report: {html_report}") + if args.open: + webbrowser.open(html_report.as_uri()) + + +if __name__ == "__main__": + main()