Parallelize the windows CI to enable us to run all tests all the time (#8868)

Split Windows CI into the following Jobs

1. Combine Build and xUnit, which published build artifact
1. Elevated CI tests
1. Unelevated CI test
1. Elevated everything else tests
1. Unelevated everything else tests
1. Package build and test
1. Verify xUnit result

## PR Context  

Before this test:

- A CI run would take ~14 minutes
- A Feature run would take ~30 minutes

After this change, it always runs all tests and takes ~20 minutes.
Job `Unelevated everything else tests` is the long poll, taking ~4 minutes longer than the next shortest job.  We should be able to get this back down to ~15 if we split that test job somehow.

.vsts-ci/templates/windows-build.yml

@@ -0,0 +1,43 @@
+parameters:
+  pool: 'Hosted VS2017'
+  jobName: 'win_build'
+
+jobs:
+- job: ${{ parameters.jobName }}
+  pool:
+    name: ${{ parameters.pool }}
+
+  displayName: Windows Build
+
+  steps:
+  - powershell: |
+      Get-ChildItem -Path env:
+    displayName: Capture environment
+    condition: succeededOrFailed()
+
+  - powershell:  Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))"
+    displayName: Set Build Name for Non-PR
+    condition: ne(variables['Build.Reason'], 'PullRequest')
+
+  - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
+
+  - powershell: |
+      [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+      Import-Module .\tools\ci.psm1
+      Invoke-AppveyorInstall
+    displayName: Bootstrap
+    condition: succeededOrFailed()
+
+  - powershell: |
+      Import-Module .\tools\ci.psm1
+      Invoke-AppveyorBuild
+    displayName: Build
+    condition: succeeded()
+
+  - powershell: |
+      Import-Module .\tools\ci.psm1
+      Restore-PSOptions
+      Invoke-CIxUnit -SkipFailing
+    displayName: xUnit Tests
+    condition: succeeded()
+

.vsts-ci/templates/windows-packaging.yml

@@ -0,0 +1,30 @@
+parameters:
+  pool: 'Hosted VS2017'
+  jobName: 'win_packaging'
+
+jobs:
+- job: ${{ parameters.jobName }}
+  pool:
+    name: ${{ parameters.pool }}
+
+  displayName: Windows Packaging
+
+  steps:
+  - powershell: |
+      Get-ChildItem -Path env:
+    displayName: Capture environment
+    condition: succeededOrFailed()
+
+  - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
+
+  - powershell: |
+      [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+      Import-Module .\tools\ci.psm1
+      Invoke-AppveyorInstall
+    displayName: Bootstrap
+    condition: succeededOrFailed()
+
+  - powershell: |
+      Import-Module .\tools\ci.psm1
+      Invoke-AppveyorFinish -NuGetKey $(NUGET_KEY)
+    displayName: Build and Test Package

.vsts-ci/templates/windows-test.yml

@@ -0,0 +1,51 @@
+parameters:
+  pool: 'Hosted VS2017'
+  parentJobs: []
+  purpose: ''
+  tagSet: 'CI'
+
+jobs:
+- job: win_test_${{ parameters.purpose }}_${{ parameters.tagSet }}
+  dependsOn:
+    ${{ parameters.parentJobs }}
+  pool:
+    name: ${{ parameters.pool }}
+
+  displayName: Windows Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }}
+
+  steps:
+  - powershell: |
+      Get-ChildItem -Path env:
+    displayName: Capture environment
+    condition: succeededOrFailed()
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download build artifacts'
+    inputs:
+      downloadType: specific
+      itemPattern: |
+        build/**/*
+      downloadPath: '$(System.ArtifactsDirectory)'
+
+  - powershell: |
+      dir "$(System.ArtifactsDirectory)\*" -Recurse
+    displayName: 'Capture artifacts directory'
+    continueOnError: true
+
+  - powershell: |
+      [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+      Import-Module .\tools\ci.psm1
+      Invoke-AppveyorInstall
+    displayName: Bootstrap
+    condition: succeededOrFailed()
+
+  - powershell: |
+      Import-Module .\tools\ci.psm1
+      Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
+      $options = (Get-PSOptions)
+      $path = split-path -path $options.Output
+      $rootPath = split-Path -path $path
+      Expand-Archive -Path '$(System.ArtifactsDirectory)\build\build.zip' -DestinationPath $rootPath -Force
+      Invoke-AppveyorTest -Purpose '${{ parameters.purpose }}' -TagSet '${{ parameters.tagSet }}'
+    displayName: Test
+    condition: succeeded()

.vsts-ci/windows.yml

@@ -23,15 +23,6 @@ pr:
     exclude:
     - /tools/releaseBuild/**/*
     - /.vsts-ci/misc-analysis.yml
-queue:
-  name: Hosted VS2017
-  parallel: 2 # Limit to two agents at a time
-  matrix:
-    UnelevatedPesterTests:
-      Purpose: UnelevatedPesterTests
-    ElevatedPesterTests_xUnit_Packaging:
-      Purpose: ElevatedPesterTests_xUnit_Packaging
-  timeoutInMinutes: 120
 
 variables:
   GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'"
@@ -44,49 +35,63 @@ resources:
 - repo: self
   clean: true
 
-steps:
-  - powershell: |
-      Get-ChildItem -Path env:
-    displayName: Capture environment
-    condition: succeededOrFailed()
+jobs:
+- template: templates/windows-build.yml
 
-  - powershell:  Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))"
-    displayName: Set Build Name for Non-PR
-    condition: ne(variables['Build.Reason'], 'PullRequest')
+- template: templates/windows-test.yml
+  parameters:
+    purpose: UnelevatedPesterTests
+    tagSet: CI
+    parentJobs:
+      - win_build
 
-  - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
+- template: templates/windows-test.yml
+  parameters:
+    purpose: ElevatedPesterTests
+    tagSet: CI
+    parentJobs:
+      - win_build
+
+- template: templates/windows-test.yml
+  parameters:
+    purpose: UnelevatedPesterTests
+    tagSet: Others
+    parentJobs:
+      - win_build
+
+- template: templates/windows-test.yml
+  parameters:
+    purpose: ElevatedPesterTests
+    tagSet: Others
+    parentJobs:
+      - win_build
+
+- template: templates/windows-packaging.yml
+
+- job: verify_xunit
+  displayName: Verify xUnit Results
+  pool:
+    name: 'Hosted VS2017'
+  dependsOn:
+    - win_build
+  steps:
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download build artifacts'
+    inputs:
+      downloadType: specific
+      itemPattern: |
+        xunit/**/*
+      downloadPath: '$(System.ArtifactsDirectory)'
 
   - powershell: |
-      [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
-      Import-Module .\tools\ci.psm1
-      Invoke-AppveyorInstall
-    displayName: Bootstrap
-    condition: succeededOrFailed()
+      dir "$(System.ArtifactsDirectory)\*" -Recurse
+    displayName: 'Capture artifacts directory'
+    continueOnError: true
 
   - powershell: |
       Import-Module .\tools\ci.psm1
-      Invoke-AppveyorBuild
-      Save-PSOptions
-    displayName: Build
-    condition: succeeded()
+      $ParallelXUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\ParallelXUnitTestResults.xml"
 
-  - powershell: |
-      Import-Module .\tools\ci.psm1
-      Restore-PSOptions
-      Invoke-AppveyorTest -Purpose '$(Purpose)'
+      Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
     displayName: Test
     condition: succeeded()
-
-  - powershell: |
-      Import-Module .\tools\ci.psm1
-      Restore-PSOptions
-      Invoke-AppveyorAfterTest
-    displayName: AfterTest
-    condition: succeededOrFailed()
-
-  - powershell: |
-      Import-Module .\tools\ci.psm1
-      Restore-PSOptions
-      Invoke-AppveyorFinish -NuGetKey $(NUGET_KEY)
-    displayName: Finish
-    condition: eq(variables['Purpose'], 'ElevatedPesterTests_xUnit_Packaging')

build.psm1

@@ -2744,45 +2744,6 @@ function Save-PSOptions {
     $Options | ConvertTo-Json -Depth 3 | Out-File -Encoding utf8 -FilePath $PSOptionsPath
 }
 
-# Restore PSOptions
-# Optionally remove the PSOptions file
-function Restore-PSOptions {
-    param(
-        [ValidateScript({Test-Path $_})]
-        [string]
-        $PSOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath 'psoptions.json'),
-        [switch]
-        $Remove
-    )
-
-    $options = Get-Content -Path $PSOptionsPath | ConvertFrom-Json
-
-    if($Remove)
-    {
-        # Remove PSOptions.
-        # The file is only used to set the PSOptions.
-        Remove-Item -Path $psOptionsPath
-    }
-
-    Set-PSOptions -Options $options
-}
-
-# Save PSOptions to be restored by Restore-PSOptions
-function Save-PSOptions {
-    param(
-        [ValidateScript({$parent = Split-Path $_;if($parent){Test-Path $parent}else{return $true}})]
-        [ValidateNotNullOrEmpty()]
-        [string]
-        $PSOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath 'psoptions.json'),
-
-        [ValidateNotNullOrEmpty()]
-        [object]
-        $Options = (Get-PSOptions -DefaultToNew)
-    )
-
-    $Options | ConvertTo-Json -Depth 3 | Out-File -Encoding utf8 -FilePath $PSOptionsPath
-}
-
 # Restore PSOptions
 # Optionally remove the PSOptions file
 function Restore-PSOptions {

tools/ci.psm1

@@ -191,6 +191,20 @@ function Invoke-AppVeyorBuild
     }
 
     Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -CI -ReleaseTag $releaseTag
+
+    Save-PSOptions
+
+    $options = (Get-PSOptions)
+
+    $path = split-path -path $options.Output
+
+    $psOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath '../psoptions.json')
+    $buildZipPath = (Join-Path -Path $PSScriptRoot -ChildPath '../build.zip')
+
+    Compress-Archive -Path $path -DestinationPath $buildZipPath
+
+    Push-Artifact -Path $psOptionsPath -Name 'build'
+    Push-Artifact -Path $buildZipPath -Name 'build'
 }
 
 # Implements the CI 'install' step
@@ -263,13 +277,41 @@ function Update-TestResults
     }
 }
 
+function Invoke-CIxUnit
+{
+    param(
+        [switch]
+        $SkipFailing
+    )
+    $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))
+    if(!(Test-Path "$env:CoreOutput\pwsh.exe"))
+    {
+        throw "CoreCLR pwsh.exe was not built"
+    }
+
+    $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml"
+
+    Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
+    Write-Host -ForegroundColor Green 'Uploading PSxUnit test results'
+    Update-TestResults -resultsFile $ParallelXUnitTestResultsFile
+    Push-Artifact -Path $ParallelXUnitTestResultsFile -name xunit
+
+    if(!$SkipFailing.IsPresent)
+    {
+        # Fail the build, if tests failed
+        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
+    }
+}
+
 # Implement CI 'Test_script'
 function Invoke-AppVeyorTest
 {
     [CmdletBinding()]
     param(
-        [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests_xUnit_Packaging')]
-        [string] $Purpose
+        [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests')]
+        [string] $Purpose,
+        [ValidateSet('CI', 'Others')]
+        [string] $TagSet
     )
     # CoreCLR
 
@@ -277,7 +319,6 @@ function Invoke-AppVeyorTest
     Write-Host -Foreground Green 'Run CoreCLR tests'
     $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml"
     $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml"
-    $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml"
     if(!(Test-Path "$env:CoreOutput\pwsh.exe"))
     {
         throw "CoreCLR pwsh.exe was not built"
@@ -286,15 +327,18 @@ function Invoke-AppVeyorTest
     # Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently
     # https://github.com/pester/Pester/issues/608
     # To work-around it, we exlude all categories, but 'CI' from the list
-    if (Test-DailyBuild)
-    {
-        $ExcludeTag = @()
-        Write-Host -Foreground Green 'Running all CoreCLR tests..'
-    }
-    else
-    {
-        $ExcludeTag = @('Slow', 'Feature', 'Scenario')
-        Write-Host -Foreground Green 'Running "CI" CoreCLR tests..'
+    switch ($TagSet) {
+        'CI' {
+            Write-Host -Foreground Green 'Running "CI" CoreCLR tests..'
+            $ExcludeTag = @('Slow', 'Feature', 'Scenario')
+        }
+        'Others' {
+            Write-Host -Foreground Green 'Running non-CI CoreCLR tests..'
+            $ExcludeTag = @('CI')
+        }
+        Default {
+            throw "Unknow TagSet: '$TagSet'"
+        }
     }
 
     # Get the experimental feature names and the tests associated with them
@@ -309,7 +353,7 @@ function Invoke-AppVeyorTest
             Tag = @()
             ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows'
         }
-        Start-PSPester @arguments -Title 'Pester Unelevated'
+        Start-PSPester @arguments -Title "Pester Unelevated - $TagSet"
         Write-Host -Foreground Green 'Upload CoreCLR Non-Admin test results'
         Update-TestResults -resultsFile $testResultsNonAdminFile
         # Fail the build, if tests failed
@@ -341,7 +385,7 @@ function Invoke-AppVeyorTest
         }
     }
 
-    if ($Purpose -eq 'ElevatedPesterTests_xUnit_Packaging') {
+    if ($Purpose -eq 'ElevatedPesterTests') {
         $arguments = @{
             Terse = $true
             Bindir = $env:CoreOutput
@@ -349,17 +393,13 @@ function Invoke-AppVeyorTest
             Tag = @('RequireAdminOnWindows')
             ExcludeTag = $ExcludeTag
         }
-        Start-PSPester @arguments -Title 'Pester Elevated'
+        Start-PSPester @arguments -Title "Pester Elevated - $TagSet"
         Write-Host -Foreground Green 'Upload CoreCLR Admin test results'
         Update-TestResults -resultsFile $testResultsAdminFile
 
-        Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
-        Write-Host -ForegroundColor Green 'Uploading PSxUnit test results'
-        Update-TestResults -resultsFile $ParallelXUnitTestResultsFile
 
         # Fail the build, if tests failed
         Test-PSPesterResults -TestResultsFile $testResultsAdminFile
-        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
 
         # Run tests with specified experimental features enabled
         foreach ($entry in $ExperimentalFeatureTests.GetEnumerator())