[release/v7.5] Update test result processing to use NUnitXml format and enhance logging for better clarity (#25344)

Co-authored-by: Travis Plunk <travis.plunk@microsoft.com>

.github/actions/test/nix/action.yml

@@ -101,7 +101,7 @@ runs:
       chmod a+x $pwshPath
       $options.Output = $pwshPath
       Set-PSOptions $options
-      Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -TitlePrefix '${{ inputs.buildName }}' -OutputFormat JUnitXml
+      Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -TitlePrefix '${{ inputs.buildName }}' -OutputFormat NUnitXml
     shell: pwsh
 
   - name: Convert, Publish, and Upload Pester Test Results

.github/actions/test/process-pester-results/action.yml

@@ -10,46 +10,13 @@ inputs:
     required: false
     default: "${{ runner.workspace }}/testResults"
     type: string
-  ctrfFolder:
-    required: false
-    default: ctrf
-    type: string
 
 runs:
   using: composite
   steps:
   - name: Log Summary
-    run: |
-      if (-not $env:GITHUB_STEP_SUMMARY) {
-        Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment."
-        exit 1
-      }
-
-      $testCaseCount = 0
-      $testErrorCount = 0
-      $testFailureCount = 0
-      $testDisabledCount = 0
-      Get-ChildItem -Path "${{ inputs.testResultsFolder }}/*.xml" -Recurse | ForEach-Object {
-        $results = [xml] (get-content $_.FullName)
-        $testCaseCount += $results.testsuites.tests
-        $testErrorCount += $results.testsuites.errors
-        $testFailureCount += $results.testsuites.failures
-        $testDisabledCount += $results.testsuites.disabled
-      }
-
-      @"
-
-      # Summary of ${{ inputs.name }}
-
-      - Total Tests: $testCaseCount
-      - Total Errors: $testErrorCount
-      - Total Failures: $testFailureCount
-      - Total Disabled: $testDisabledCount
-
-      "@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append
-
-      Write-Host "Summary written to $ENV:GITHUB_STEP_SUMMARY"
-      Get-Content $ENV:GITHUB_STEP_SUMMARY
+    run: |-
+      & "$env:GITHUB_ACTION_PATH/process-pester-results.ps1" -Name '${{ inputs.name }}' -TestResultsFolder '${{ inputs.testResultsFolder }}'
     shell: pwsh
 
   - name: Upload testResults artifact

.github/actions/test/process-pester-results/process-pester-results.ps1

@@ -0,0 +1,68 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+param(
+    [parameter(Mandatory)]
+    [string]$Name,
+    [parameter(Mandatory)]
+    [string]$TestResultsFolder
+)
+
+Import-Module "$PSScriptRoot/../../../../build.psm1"
+
+if (-not $env:GITHUB_STEP_SUMMARY) {
+    Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment."
+    exit 1
+}
+
+$testCaseCount = 0
+$testErrorCount = 0
+$testFailureCount = 0
+$testNotRunCount = 0
+$testInconclusiveCount = 0
+$testIgnoredCount = 0
+$testSkippedCount = 0
+$testInvalidCount = 0
+
+Get-ChildItem -Path "${TestResultsFolder}/*.xml" -Recurse | ForEach-Object {
+    $results = [xml] (get-content $_.FullName)
+
+    $testCaseCount += [int]$results.'test-results'.total
+    $testErrorCount += [int]$results.'test-results'.errors
+    $testFailureCount += [int]$results.'test-results'.failures
+    $testNotRunCount += [int]$results.'test-results'.'not-run'
+    $testInconclusiveCount += [int]$results.'test-results'.inconclusive
+    $testIgnoredCount += [int]$results.'test-results'.ignored
+    $testSkippedCount += [int]$results.'test-results'.skipped
+    $testInvalidCount += [int]$results.'test-results'.invalid
+}
+
+@"
+
+# Summary of $Name
+
+- Total Tests: $testCaseCount
+- Total Errors: $testErrorCount
+- Total Failures: $testFailureCount
+- Total Not Run: $testNotRunCount
+- Total Inconclusive: $testInconclusiveCount
+- Total Ignored: $testIgnoredCount
+- Total Skipped: $testSkippedCount
+- Total Invalid: $testInvalidCount
+
+"@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append
+
+Write-Log "Summary written to $ENV:GITHUB_STEP_SUMMARY"
+
+Write-LogGroupStart -Title 'Test Results'
+Get-Content $ENV:GITHUB_STEP_SUMMARY
+Write-LogGroupEnd -Title 'Test Results'
+
+if ($testErrorCount -gt 0 -or $testFailureCount -gt 0) {
+    Write-Error "There were $testErrorCount/$testFailureCount errors/failures in the test results."
+    exit 1
+}
+if ($testCaseCount -eq 0) {
+    Write-Error "No test cases were run."
+    exit 1
+}

.github/actions/test/windows/action.yml

@@ -20,15 +20,25 @@ runs:
   steps:
   - name: Capture Environment
     if: success() || failure()
-    run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose'
+    run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Environment'
+      Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose
+      Write-LogGroupEnd -Title 'Environment'
     shell: pwsh
+
   - name: Download Build Artifacts
     uses: actions/download-artifact@v4
     with:
       path: "${{ github.workspace }}"
+
   - name: Capture Artifacts Directory
     continue-on-error: true
-    run: Get-ChildItem "${{ github.workspace }}\build\*" -Recurse
+    run: |-
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Artifacts Directory'
+      Get-ChildItem "${{ github.workspace }}/build/*" -Recurse
+      Write-LogGroupEnd -Title 'Artifacts Directory'
     shell: pwsh
 
   - uses: actions/setup-dotnet@v4
@@ -38,7 +48,8 @@ runs:
   - name: Bootstrap
     shell: powershell
     run: |-
-      # Remove "Program Files\dotnet" from the env variable PATH, so old SDKs won't affect us.
+      Import-Module ./build.psm1
+      Write-LogGroupStart -Title 'Bootstrap'
       Write-Host "Old Path:"
       Write-Host $env:Path
       $dotnetPath = Join-Path $env:SystemDrive 'Program Files\dotnet'
@@ -49,6 +60,7 @@ runs:
       # Bootstrap
       Import-Module .\tools\ci.psm1
       Invoke-CIInstall
+      Write-LogGroupEnd -Title 'Bootstrap'
 
   - name: Test
     if: success()
@@ -60,7 +72,7 @@ runs:
       $path = split-path -path $options.Output
       $rootPath = split-Path -path $path
       Expand-Archive -Path '${{ github.workspace }}\build\build.zip' -DestinationPath $rootPath -Force
-      Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -OutputFormat JUnitXml
+      Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -OutputFormat NUnitXml
     shell: pwsh
 
   - name: Convert, Publish, and Upload Pester Test Results

build.psm1

@@ -1231,6 +1231,7 @@ function Get-PesterTag {
 # testing PowerShell remote custom connections.
 function Publish-CustomConnectionTestModule
 {
+    Write-LogGroupStart -Title "Publish-CustomConnectionTestModule"
     $sourcePath = "${PSScriptRoot}/test/tools/NamedPipeConnection"
     $outPath = "${PSScriptRoot}/test/tools/NamedPipeConnection/out/Microsoft.PowerShell.NamedPipeConnection"
     $publishPath = "${PSScriptRoot}/test/tools/Modules"
@@ -1255,6 +1256,8 @@ function Publish-CustomConnectionTestModule
     finally {
         Pop-Location
     }
+
+    Write-LogGroupEnd -Title "Publish-CustomConnectionTestModule"
 }
 
 function Publish-PSTestTools {
@@ -1264,6 +1267,7 @@ function Publish-PSTestTools {
         $runtime
     )
 
+    Write-LogGroupStart -Title "Publish-PSTestTools"
     Find-Dotnet
 
     $tools = @(
@@ -1335,6 +1339,7 @@ function Publish-PSTestTools {
 
     # Publish the Microsoft.PowerShell.NamedPipeConnection module
     Publish-CustomConnectionTestModule
+    Write-LogGroupEnd -Title "Publish-PSTestTools"
 }
 
 function Get-ExperimentalFeatureTests {
@@ -1822,12 +1827,16 @@ function Show-PSPesterError
         throw 'Unknown Show-PSPester parameter set'
     }
 
-    Write-Log -isError -message ("Description: " + $description)
-    Write-Log -isError -message ("Name:        " + $name)
-    Write-Log -isError -message "message:"
-    Write-Log -isError -message $message
-    Write-Log -isError -message "stack-trace:"
-    Write-Log -isError -message $stack_trace
+    # Empty line at the end is intentional formatting
+    Write-Log -isError -message @"
+Description: $description
+Name:        $name
+message:
+$message
+stack-trace:
+$stack_trace
+
+"@
 
 }
 
@@ -1867,13 +1876,17 @@ function Test-XUnitTestResults
         $message = $failure.failure.message
         $stack_trace = $failure.failure.'stack-trace'
 
-        Write-Log -isError -message ("Description: " + $description)
-        Write-Log -isError -message ("Name:        " + $name)
-        Write-Log -isError -message "message:"
-        Write-Log -isError -message $message
-        Write-Log -isError -message "stack-trace:"
-        Write-Log -isError -message $stack_trace
-        Write-Log -isError -message " "
+        # Empty line at the end is intentional formatting
+        Write-Log -isError -message         @"
+        Description: $description
+        Name:        $name
+        message:
+        $message
+        stack-trace:
+        $stack_trace
+
+"@
+
     }
 
     throw "$($results.assemblies.assembly.failed) tests failed"
@@ -1909,7 +1922,8 @@ function Test-PSPesterResults
         $x = [xml](Get-Content -Raw $testResultsFile)
         if ([int]$x.'test-results'.failures -gt 0)
         {
-            Write-Log -isError -message "TEST FAILURES"
+            Write-LogGroupStart -Title 'TEST FAILURES'
+
             # switch between methods, SelectNode is not available on dotnet core
             if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] )
             {
@@ -1923,6 +1937,8 @@ function Test-PSPesterResults
             {
                 Show-PSPesterError -testFailure $testfail
             }
+
+            Write-LogGroupEnd -Title 'TEST FAILURES'
             throw "$($x.'test-results'.failures) tests in $TestArea failed"
         }
     }
@@ -1943,11 +1959,12 @@ function Test-PSPesterResults
         }
         elseif ($ResultObject.FailedCount -gt 0)
         {
-            Write-Log -isError -message 'TEST FAILURES'
+            Write-LogGroupStart -Title 'TEST FAILURES'
 
             $ResultObject.TestResult | Where-Object {$_.Passed -eq $false} | ForEach-Object {
                 Show-PSPesterError -testFailureObject $_
             }
+            Write-LogGroupEnd -Title 'TEST FAILURES'
 
             throw "$($ResultObject.FailedCount) tests in $TestArea failed"
         }
@@ -2694,10 +2711,12 @@ function script:Write-Log
     )
     if ($isError)
     {
-        Write-Host -Foreground Red $message
-        if($env:GITHUB_WORKFLOW)
-        {
-            Write-Host "::error::${message}"
+        if ($env:GITHUB_WORKFLOW) {
+            # https://github.com/actions/toolkit/issues/193#issuecomment-605394935
+            $escapedMessage = $message -replace "`n", "%0A" -replace "`r"
+            Write-Host "::error::${escapedMessage}"
+        } else {
+            Write-Host -Foreground Red $message
         }
     }
     else