trailofbits-buttercup/.github/workflows/tests.yml

name: Unit tests

on:
  push:
    branches:
      - main
    # Always run full test suite on main branch
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    strategy:
      fail-fast: false  # Continue running other components even if one fails
      matrix:
        include:
          - component: common
            coverage_module: buttercup.common
            python: "3.12"
          - component: orchestrator
            coverage_module: buttercup.orchestrator
            python: "3.12"
          - component: program-model
            coverage_module: buttercup.program_model
            python: "3.12"
          - component: seed-gen
            coverage_module: buttercup.seed_gen
            python: "3.12"
          - component: patcher
            coverage_module: buttercup.patcher
            python: "3.12"
          - component: fuzzer
            coverage_module: buttercup.fuzzer
            python: "3.12"

    runs-on: ubuntu-latest
    # Removed if: matrix.should_run since we're not using path filtering right now
    services:
      redis:
        image: redis
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379

    steps:
      - uses: actions/checkout@v5
        with:
          persist-credentials: false
          submodules: true

      - name: Install uv
        uses: astral-sh/setup-uv@v6

      - name: Setup uv cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/uv
            ~/.local/share/uv
          key: ${{ runner.os }}-uv-${{ matrix.component }}-${{ hashFiles(format('{0}/uv.lock', matrix.component)) }}
          restore-keys: |
            ${{ runner.os }}-uv-${{ matrix.component }}-
            ${{ runner.os }}-uv-

      - name: Download Wasm runtime
        run: wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.12.0%2B20231211-040d5a6/python-3.12.0.wasm
        if: matrix.component == 'seed-gen'
        working-directory: seed-gen

      - name: Install dependencies for program-model, seed-gen, and patcher
        if: matrix.component == 'program-model' || matrix.component == 'seed-gen' || matrix.component == 'patcher'
        run: |
          sudo apt-get update
          sudo apt-get install -y codequery ripgrep
          make install-cscope

      - name: Install minimal dependencies
        if: matrix.component != 'program-model' && matrix.component != 'seed-gen' && matrix.component != 'patcher' && matrix.component != 'fuzzer'
        run: |
          sudo apt-get update
          sudo apt-get install -y ripgrep
      
      # Fuzzer only needs ripgrep, no codequery
      - name: Install fuzzer dependencies
        if: matrix.component == 'fuzzer'
        run: |
          sudo apt-get update
          sudo apt-get install -y ripgrep

      - name: Prepare environment
        run: |
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update
          sudo mkdir -p /crs_scratch
          sudo chmod -R 777 /crs_scratch

      - name: Setup ${{ matrix.component }} component
        run: |
          uv sync --all-extras --frozen
          # Install test reporting tools in isolated environment
          # This avoids adding them to every component's dependencies
          uv pip install --isolated pytest-html>=4.1.1 pytest-cov>=6.0.0
        working-directory: ${{ matrix.component }}

      - name: Run tests on ${{ matrix.component }} component
        run: |
          uv run --frozen pytest -svv ${{ matrix.pytest_args }} \
            --junit-xml=test-results.xml \
            --html=test-report.html \
            --self-contained-html \
            --cov=${{ matrix.coverage_module }} \
            --cov-report=xml \
            --cov-report=html \
            --cov-report=term
        env:
          PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
        working-directory: ${{ matrix.component }}

      - name: Generate test summary
        if: always()
        run: |
          echo "### Test Results: ${{ matrix.component }}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          if [ -f ${{ matrix.component }}/test-results.xml ]; then
            python -c "
          import xml.etree.ElementTree as ET
          tree = ET.parse('${{ matrix.component }}/test-results.xml')
          root = tree.getroot()
          tests = root.get('tests', '0')
          failures = root.get('failures', '0')
          errors = root.get('errors', '0')
          skipped = root.get('skipped', '0')
          time = root.get('time', '0')
          print(f'- **Total Tests**: {tests}')
          print(f'- **Passed**: {int(tests) - int(failures) - int(errors) - int(skipped)}')
          print(f'- **Failed**: {failures}')
          print(f'- **Errors**: {errors}')
          print(f'- **Skipped**: {skipped}')
          print(f'- **Duration**: {float(time):.2f}s')
          " >> $GITHUB_STEP_SUMMARY
          else
            echo "No test results found" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ matrix.component }}-py${{ matrix.python }}
          path: |
            ${{ matrix.component }}/test-results.xml
            ${{ matrix.component }}/test-report.html
            ${{ matrix.component }}/coverage.xml
            ${{ matrix.component }}/htmlcov/
          retention-days: 30

      # Coverage will be uploaded in a separate job after all tests complete

  # Consolidated coverage upload after all tests complete
  coverage-upload:
    needs: [test]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Download all coverage reports
        uses: actions/download-artifact@v5
        with:
          pattern: test-results-*
          path: coverage-reports

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        with:
          directory: coverage-reports
          files: '*/coverage.xml,**/coverage.xml'
          fail_ci_if_error: false
          verbose: true