[release/v7.4] Make GitHub Workflows work in the internal mirror (#25409)

Co-authored-by: Travis Plunk <travis.plunk@microsoft.com>

.github/actions/infrastructure/path-filters/action.yml

@@ -0,0 +1,88 @@
+name: Path Filters
+description: 'Path Filters'
+inputs:
+  GITHUB_TOKEN:
+    description: 'GitHub token'
+    required: true
+outputs:
+  source:
+    description: 'Source code changes (composite of all changes)'
+    value: ${{ steps.filter.outputs.source }}
+  githubChanged:
+    description: 'GitHub workflow changes'
+    value: ${{ steps.filter.outputs.githubChanged }}
+  toolsChanged:
+    description: 'Tools changes'
+    value: ${{ steps.filter.outputs.toolsChanged }}
+  propsChanged:
+    description: 'Props changes'
+    value: ${{ steps.filter.outputs.propsChanged }}
+  testsChanged:
+    description: 'Tests changes'
+    value: ${{ steps.filter.outputs.testsChanged }}
+  mainSourceChanged:
+    description: 'Main source code changes (any changes in src/)'
+    value: ${{ steps.filter.outputs.mainSourceChanged }}
+  buildModuleChanged:
+    description: 'Build module changes'
+    value: ${{ steps.filter.outputs.buildModuleChanged }}
+runs:
+  using: composite
+  steps:
+    - name: Check if GitHubWorkflowChanges is present
+      id: filter
+      uses: actions/github-script@v7.0.1
+      with:
+        github-token: ${{ inputs.GITHUB_TOKEN }}
+        script: |
+          // Fetch the list of files changed in the PR
+          let files = [];
+          let page = 1;
+          let fetchedFiles;
+          do {
+            fetchedFiles = await github.rest.pulls.listFiles({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+              per_page: 100,
+              page: page++
+            });
+            files = files.concat(fetchedFiles.data);
+          } while (fetchedFiles.data.length > 0);
+
+          const actionsChanged = files.some(file => file.filename.startsWith('.github/actions'));
+          const workflowsChanged = files.some(file => file.filename.startsWith('.github/workflows'));
+          const githubChanged = actionsChanged || workflowsChanged;
+
+          const toolsCiPsm1Changed = files.some(file => file.filename.startsWith('tools/ci.psm1'));
+          const toolsBuildCommonChanged = files.some(file => file.filename.startsWith('tools/buildCommon/'));
+          const toolsChanged = toolsCiPsm1Changed || toolsBuildCommonChanged;
+
+          const propsChanged = files.some(file => file.filename.endsWith('.props'));
+
+          const testsChanged = files.some(file => file.filename.startsWith('test/powershell/') || file.filename.startsWith('test/tools/') || file.filename.startsWith('test/xUnit/'));
+
+          const mainSourceChanged = files.some(file => file.filename.startsWith('src/'));
+
+          const buildModuleChanged = files.some(file => file.filename.startsWith('build.psm1'));
+
+          const source = mainSourceChanged || toolsChanged || githubChanged || propsChanged || testsChanged;
+
+          core.setOutput('toolsChanged', toolsChanged);
+          core.setOutput('githubChanged', githubChanged);
+          core.setOutput('propsChanged', propsChanged);
+          core.setOutput('testsChanged', testsChanged);
+          core.setOutput('mainSourceChanged', mainSourceChanged);
+          core.setOutput('buildModuleChanged', buildModuleChanged);
+          core.setOutput('source', source);
+
+    - name: Capture outputs
+      run: |
+        Write-Verbose -Verbose "source: ${{ steps.filter.outputs.source }}"
+        Write-Verbose -Verbose "github: ${{ steps.filter.outputs.githubChanged }}"
+        Write-Verbose -Verbose "tools: ${{ steps.filter.outputs.toolsChanged }}"
+        Write-Verbose -Verbose "props: ${{ steps.filter.outputs.propsChanged }}"
+        Write-Verbose -Verbose "tests: ${{ steps.filter.outputs.testsChanged }}"
+        Write-Verbose -Verbose "mainSource: ${{ steps.filter.outputs.mainSourceChanged }}"
+        Write-Verbose -Verbose "buildModule: ${{ steps.filter.outputs.buildModuleChanged }}"
+      shell: pwsh

.github/actions/test/nix/action.yml

@@ -20,26 +20,39 @@ runs:
   steps:
   - name: Capture Environment
     if: success() || failure()
-    run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose'
+    run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Environment'
+      Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose
+      Write-LogGroupEnd -Title 'Environment'
     shell: pwsh
+
   - name: Download Build Artifacts
     uses: actions/download-artifact@v4
     with:
       path: "${{ github.workspace }}"
+
   - name: Capture Artifacts Directory
     continue-on-error: true
-    run: Get-ChildItem "${{ github.workspace }}/build/*" -Recurse
+    run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Artifacts Directory'
+      Get-ChildItem "${{ github.workspace }}/build/*" -Recurse
+      Write-LogGroupEnd -Title 'Artifacts Directory'
     shell: pwsh
-    
+
   - uses: actions/setup-dotnet@v4
     with:
       global-json-file: ./global.json
-      
+
   - name: Bootstrap
     shell: pwsh
     run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Bootstrap'
       Import-Module ./tools/ci.psm1
       Invoke-CIInstall -SkipUser
+      Write-LogGroupEnd -Title 'Bootstrap'
 
   - name: Extract Files
     uses: actions/github-script@v7.0.0
@@ -68,7 +81,11 @@ runs:
 
   - name: Capture Extracted Build ZIP
     continue-on-error: true
-    run: Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue
+    run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Extracted Build ZIP'
+      Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue
+      Write-LogGroupEnd -Title 'Extracted Build ZIP'
     shell: pwsh
 
   - name: Test

.github/actions/test/process-pester-results/action.yml

@@ -18,36 +18,39 @@ inputs:
 runs:
   using: composite
   steps:
-  - name: Convert JUnit to CTRF
-    run: |-
-      Get-ChildItem -Path "${{ inputs.testResultsFolder }}/*.xml" -Recurse | ForEach-Object {
-        npx --yes junit-to-ctrf $_.FullName --output ./${{ inputs.ctrfFolder }}/$($_.BaseName).json --tool Pester
+  - name: Log Summary
+    run: |
+      if (-not $env:GITHUB_STEP_SUMMARY) {
+        Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment."
+        exit 1
       }
-    shell: pwsh
 
-  # this task only takes / as directory separators
-  - name: Publish Test Report
-    uses: ctrf-io/github-test-reporter@v1
-    with:
-      report-path: './${{ inputs.ctrfFolder }}/*.json'
-      exit-on-fail: true
-      summary-report: true
-      test-report: false
-      test-list-report: false
-      failed-report: false
-      fail-rate-report: false
-      flaky-report: false
-      flaky-rate-report: false
-      failed-folded-report: true
-      previous-results-report: false
-      ai-report: true
-      skipped-report: false
-      suite-folded-report: false
-      suite-list-report: false
-      pull-request-report: false
-      commit-report: false
-      custom-report: false
-    if: always()
+      $testCaseCount = 0
+      $testErrorCount = 0
+      $testFailureCount = 0
+      $testDisabledCount = 0
+      Get-ChildItem -Path "${{ inputs.testResultsFolder }}/*.xml" -Recurse | ForEach-Object {
+        $results = [xml] (get-content $_.FullName)
+        $testCaseCount += $results.testsuites.tests
+        $testErrorCount += $results.testsuites.errors
+        $testFailureCount += $results.testsuites.failures
+        $testDisabledCount += $results.testsuites.disabled
+      }
+
+      @"
+
+      # Summary of ${{ inputs.name }}
+
+      - Total Tests: $testCaseCount
+      - Total Errors: $testErrorCount
+      - Total Failures: $testFailureCount
+      - Total Disabled: $testDisabledCount
+
+      "@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append
+
+      Write-Host "Summary written to $ENV:GITHUB_STEP_SUMMARY"
+      Get-Content $ENV:GITHUB_STEP_SUMMARY
+    shell: pwsh
 
   - name: Upload testResults artifact
     if: always()
@@ -55,10 +58,3 @@ runs:
     with:
       name: junit-pester-${{ inputs.name }}
       path: ${{ runner.workspace }}/testResults
-
-  - name: Upload ctrf artifact
-    if: always()
-    uses: actions/upload-artifact@v4
-    with:
-      name: ctrf-pester-${{ inputs.name }}
-      path: ${{ inputs.ctrfFolder }}

.github/workflows/linux-ci.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
     paths:
       - "**"
       - "!.github/ISSUE_TEMPLATE/**"
@@ -19,6 +20,7 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
 # Path filters for PRs need to go into the changes job
 
 concurrency:
@@ -43,30 +45,22 @@ jobs:
     # Required permissions
     permissions:
       pull-requests: read
+      contents: read
+
     # Set job outputs to values from filter step
     outputs:
       source: ${{ steps.filter.outputs.source }}
     steps:
       - name: checkout
-        uses: actions/checkout@v4.1.0
-
-        # For pull requests it's not necessary to checkout the code
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0
-        id: filter
+        uses: actions/checkout@v4
         with:
-          list-files: json
-          filters: .github/action-filters.yml
+          persist-credentials: false
 
-      - name: Capture outputs
-        run: |
-          "source: ${{ steps.filter.outputs.source }}"
-          "github: ${{ steps.filter.outputs.github }}"
-          "tools: ${{ steps.filter.outputs.tools }}"
-          "props: ${{ steps.filter.outputs.props }}"
-          "tests: ${{ steps.filter.outputs.tests }}"
-          "mainSource: ${{ steps.filter.outputs.mainSource }}"
-          "buildModule: ${{ steps.filter.outputs.buildModule }}"
-        shell: pwsh
+      - name: Change Detection
+        id: filter
+        uses: "./.github/actions/infrastructure/path-filters"
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   ci_build:
     name: Build PowerShell

.github/workflows/macos-ci.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
     paths:
       - "**"
       - "!.github/ISSUE_TEMPLATE/**"
@@ -17,6 +18,7 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
 # Path filters for PRs need to go into the changes job
 
 concurrency:
@@ -34,6 +36,7 @@ env:
   __SuppressAnsiEscapeSequences: 1
   nugetMultiFeedWarnLevel: none
   system_debug: 'false'
+
 jobs:
   changes:
     name: Change Detection
@@ -42,6 +45,8 @@ jobs:
     # Required permissions
     permissions:
       pull-requests: read
+      contents: read
+
     # Set job outputs to values from filter step
     outputs:
       source: ${{ steps.filter.outputs.source }}
@@ -49,23 +54,11 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4.1.0
 
-        # For pull requests it's not necessary to checkout the code
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0
+      - name: Change Detection
         id: filter
+        uses: "./.github/actions/infrastructure/path-filters"
         with:
-          list-files: json
-          filters: .github/action-filters.yml
-
-      - name: Capture outputs
-        run: |
-          "source: ${{ steps.filter.outputs.source }}"
-          "github: ${{ steps.filter.outputs.github }}"
-          "tools: ${{ steps.filter.outputs.tools }}"
-          "props: ${{ steps.filter.outputs.props }}"
-          "tests: ${{ steps.filter.outputs.tests }}"
-          "mainSource: ${{ steps.filter.outputs.mainSource }}"
-          "buildModule: ${{ steps.filter.outputs.buildModule }}"
-        shell: pwsh
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   ci_build:
     name: Build PowerShell

.github/workflows/windows-ci.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
     paths:
       - "**"
       - "!.vsts-ci/misc-analysis.yml"
@@ -16,6 +17,8 @@ on:
     branches:
       - master
       - release/**
+      - github-mirror
+
 # Path filters for PRs need to go into the changes job
 
 concurrency:
@@ -43,6 +46,8 @@ jobs:
     # Required permissions
     permissions:
       pull-requests: read
+      contents: read
+
     # Set job outputs to values from filter step
     outputs:
       source: ${{ steps.filter.outputs.source }}
@@ -50,23 +55,11 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4.1.0
 
-        # For pull requests it's not necessary to checkout the code
-      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0
+      - name: Change Detection
         id: filter
+        uses: "./.github/actions/infrastructure/path-filters"
         with:
-          list-files: json
-          filters: .github/action-filters.yml
-
-      - name: Capture outputs
-        run: |
-          "source: ${{ steps.filter.outputs.source }}"
-          "github: ${{ steps.filter.outputs.github }}"
-          "tools: ${{ steps.filter.outputs.tools }}"
-          "props: ${{ steps.filter.outputs.props }}"
-          "tests: ${{ steps.filter.outputs.tests }}"
-          "mainSource: ${{ steps.filter.outputs.mainSource }}"
-          "buildModule: ${{ steps.filter.outputs.buildModule }}"
-        shell: pwsh
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   ci_build:
     name: Build PowerShell

build.psm1

@@ -179,6 +179,8 @@ function Get-EnvironmentInformation
         $environment += @{'IsUbuntu16' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '16.04'}
         $environment += @{'IsUbuntu18' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '18.04'}
         $environment += @{'IsUbuntu20' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '20.04'}
+        $environment += @{'IsUbuntu22' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '22.04'}
+        $environment += @{'IsUbuntu24' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '24.04'}
         $environment += @{'IsCentOS' = $LinuxInfo.ID -match 'centos' -and $LinuxInfo.VERSION_ID -match '7'}
         $environment += @{'IsFedora' = $LinuxInfo.ID -match 'fedora' -and $LinuxInfo.VERSION_ID -ge 24}
         $environment += @{'IsOpenSUSE' = $LinuxInfo.ID -match 'opensuse'}
@@ -2686,6 +2688,10 @@ function script:Write-Log
     if ($isError)
     {
         Write-Host -Foreground Red $message
+        if($env:GITHUB_WORKFLOW)
+        {
+            Write-Host "::error::${message}"
+        }
     }
     else
     {
@@ -2694,6 +2700,59 @@ function script:Write-Log
     #reset colors for older package to at return to default after error message on a compilation error
     [console]::ResetColor()
 }
+
+function script:Write-LogGroup {
+    param
+    (
+        [Parameter(Position = 0, Mandatory)]
+        [ValidateNotNullOrEmpty()]
+        [string[]] $Message,
+        [Parameter(Mandatory)]
+        [string] $Title
+    )
+
+
+    Write-LogGroupStart -Title $Title
+
+    foreach ($line in $Message) {
+        Write-Log -Message $line
+    }
+
+    Write-LogGroupEnd -Title $Title
+}
+
+$script:logGroupColor = [System.ConsoleColor]::Cyan
+
+function script:Write-LogGroupStart {
+    param
+    (
+        [Parameter(Mandatory)]
+        [string] $Title
+    )
+
+    if ($env:GITHUB_WORKFLOW) {
+        Write-Host "::group::${Title}"
+    }
+    else {
+        Write-Host -ForegroundColor $script:logGroupColor "=== BEGIN: $Title ==="
+    }
+}
+
+function script:Write-LogGroupEnd {
+    param
+    (
+        [Parameter(Mandatory)]
+        [string] $Title
+    )
+
+    if ($env:GITHUB_WORKFLOW) {
+        Write-Host "::endgroup::"
+    }
+    else {
+        Write-Host -ForegroundColor $script:logGroupColor "==== END: $Title ===="
+    }
+}
+
 function script:precheck([string]$command, [string]$missedMessage) {
     $c = Get-Command $command -ErrorAction Ignore
     if (-not $c) {
@@ -3622,22 +3681,50 @@ function Set-PipelineNugetAuthentication {
 
 function Set-CorrectLocale
 {
+    Write-LogGroupStart -Title "Set-CorrectLocale"
+
     if (-not $IsLinux)
     {
+        Write-LogGroupEnd -Title "Set-CorrectLocale"
         return
     }
 
     $environment = Get-EnvironmentInformation
-    if ($environment.IsUbuntu -and $environment.IsUbuntu20)
-    {
+    if ($environment.IsUbuntu16 -or $environment.IsUbuntu18) {
+        Write-Verbose -Message "Don't set locale before Ubuntu 20" -Verbose
+        Write-LogGroupEnd -Title "Set-CorrectLocale"
+        Write-Locale
+        return
+    }
+
+    if ($environment.IsUbuntu) {
+        Write-Log -Message "Setting locale to en_US.UTF-8"
         $env:LC_ALL = 'en_US.UTF-8'
         $env:LANG = 'en_US.UTF-8'
         sudo locale-gen $env:LANG
-        sudo update-locale
+        if ($environment.IsUbuntu20) {
+            Write-Log -Message "Updating locale for Ubuntu 20"
+            sudo update-locale
+        } else {
+            Write-Log -Message "Updating locale for Ubuntu 22 and newer"
+            sudo update-locale LANG=$env:LANG LC_ALL=$env:LC_ALL
+        }
+    }
+
+    Write-LogGroupEnd -Title "Set-CorrectLocale"
+    Write-Locale
+
+}
+
+function Write-Locale {
+    if (-not $IsLinux -and -not $IsMacOS) {
+        Write-Verbose -Message "only supported on Linux and macOS" -Verbose
+        return
     }
 
     # Output the locale to log it
-    locale
+    $localOutput = & locale
+    Write-LogGroup -Title "Capture Locale" -Message $localOutput
 }
 
 function Install-AzCopy {