diff --git a/.vsts-ci/templates/windows-build.yml b/.vsts-ci/templates/windows-build.yml new file mode 100644 index 0000000000..5f6faa7f68 --- /dev/null +++ b/.vsts-ci/templates/windows-build.yml @@ -0,0 +1,43 @@ +parameters: + pool: 'Hosted VS2017' + jobName: 'win_build' + +jobs: +- job: ${{ parameters.jobName }} + pool: + name: ${{ parameters.pool }} + + displayName: Windows Build + + steps: + - powershell: | + Get-ChildItem -Path env: + displayName: Capture environment + condition: succeededOrFailed() + + - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" + displayName: Set Build Name for Non-PR + condition: ne(variables['Build.Reason'], 'PullRequest') + + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + + - powershell: | + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + Import-Module .\tools\ci.psm1 + Invoke-AppveyorInstall + displayName: Bootstrap + condition: succeededOrFailed() + + - powershell: | + Import-Module .\tools\ci.psm1 + Invoke-AppveyorBuild + displayName: Build + condition: succeeded() + + - powershell: | + Import-Module .\tools\ci.psm1 + Restore-PSOptions + Invoke-CIxUnit -SkipFailing + displayName: xUnit Tests + condition: succeeded() + diff --git a/.vsts-ci/templates/windows-packaging.yml b/.vsts-ci/templates/windows-packaging.yml new file mode 100644 index 0000000000..7b974f3844 --- /dev/null +++ b/.vsts-ci/templates/windows-packaging.yml @@ -0,0 +1,30 @@ +parameters: + pool: 'Hosted VS2017' + jobName: 'win_packaging' + +jobs: +- job: ${{ parameters.jobName }} + pool: + name: ${{ parameters.pool }} + + displayName: Windows Packaging + + steps: + - powershell: | + Get-ChildItem -Path env: + displayName: Capture environment + condition: succeededOrFailed() + + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + + - powershell: | + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + Import-Module .\tools\ci.psm1 + Invoke-AppveyorInstall + displayName: Bootstrap + condition: succeededOrFailed() + + - powershell: | + Import-Module .\tools\ci.psm1 + Invoke-AppveyorFinish -NuGetKey $(NUGET_KEY) + displayName: Build and Test Package diff --git a/.vsts-ci/templates/windows-test.yml b/.vsts-ci/templates/windows-test.yml new file mode 100644 index 0000000000..b4f68e5ee5 --- /dev/null +++ b/.vsts-ci/templates/windows-test.yml @@ -0,0 +1,51 @@ +parameters: + pool: 'Hosted VS2017' + parentJobs: [] + purpose: '' + tagSet: 'CI' + +jobs: +- job: win_test_${{ parameters.purpose }}_${{ parameters.tagSet }} + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: ${{ parameters.pool }} + + displayName: Windows Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} + + steps: + - powershell: | + Get-ChildItem -Path env: + displayName: Capture environment + condition: succeededOrFailed() + + - task: DownloadBuildArtifacts@0 + displayName: 'Download build artifacts' + inputs: + downloadType: specific + itemPattern: | + build/**/* + downloadPath: '$(System.ArtifactsDirectory)' + + - powershell: | + dir "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture artifacts directory' + continueOnError: true + + - powershell: | + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + Import-Module .\tools\ci.psm1 + Invoke-AppveyorInstall + displayName: Bootstrap + condition: succeededOrFailed() + + - powershell: | + Import-Module .\tools\ci.psm1 + Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' + $options = (Get-PSOptions) + $path = split-path -path $options.Output + $rootPath = split-Path -path $path + Expand-Archive -Path '$(System.ArtifactsDirectory)\build\build.zip' -DestinationPath $rootPath -Force + Invoke-AppveyorTest -Purpose '${{ parameters.purpose }}' -TagSet '${{ parameters.tagSet }}' + displayName: Test + condition: succeeded() diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index f27eb15055..dab9f6b295 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -23,15 +23,6 @@ pr: exclude: - /tools/releaseBuild/**/* - /.vsts-ci/misc-analysis.yml -queue: - name: Hosted VS2017 - parallel: 2 # Limit to two agents at a time - matrix: - UnelevatedPesterTests: - Purpose: UnelevatedPesterTests - ElevatedPesterTests_xUnit_Packaging: - Purpose: ElevatedPesterTests_xUnit_Packaging - timeoutInMinutes: 120 variables: GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" @@ -44,49 +35,63 @@ resources: - repo: self clean: true -steps: - - powershell: | - Get-ChildItem -Path env: - displayName: Capture environment - condition: succeededOrFailed() +jobs: +- template: templates/windows-build.yml - - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" - displayName: Set Build Name for Non-PR - condition: ne(variables['Build.Reason'], 'PullRequest') +- template: templates/windows-test.yml + parameters: + purpose: UnelevatedPesterTests + tagSet: CI + parentJobs: + - win_build - - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +- template: templates/windows-test.yml + parameters: + purpose: ElevatedPesterTests + tagSet: CI + parentJobs: + - win_build + +- template: templates/windows-test.yml + parameters: + purpose: UnelevatedPesterTests + tagSet: Others + parentJobs: + - win_build + +- template: templates/windows-test.yml + parameters: + purpose: ElevatedPesterTests + tagSet: Others + parentJobs: + - win_build + +- template: templates/windows-packaging.yml + +- job: verify_xunit + displayName: Verify xUnit Results + pool: + name: 'Hosted VS2017' + dependsOn: + - win_build + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download build artifacts' + inputs: + downloadType: specific + itemPattern: | + xunit/**/* + downloadPath: '$(System.ArtifactsDirectory)' - powershell: | - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 - Import-Module .\tools\ci.psm1 - Invoke-AppveyorInstall - displayName: Bootstrap - condition: succeededOrFailed() + dir "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture artifacts directory' + continueOnError: true - powershell: | Import-Module .\tools\ci.psm1 - Invoke-AppveyorBuild - Save-PSOptions - displayName: Build - condition: succeeded() + $ParallelXUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\ParallelXUnitTestResults.xml" - - powershell: | - Import-Module .\tools\ci.psm1 - Restore-PSOptions - Invoke-AppveyorTest -Purpose '$(Purpose)' + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile displayName: Test condition: succeeded() - - - powershell: | - Import-Module .\tools\ci.psm1 - Restore-PSOptions - Invoke-AppveyorAfterTest - displayName: AfterTest - condition: succeededOrFailed() - - - powershell: | - Import-Module .\tools\ci.psm1 - Restore-PSOptions - Invoke-AppveyorFinish -NuGetKey $(NUGET_KEY) - displayName: Finish - condition: eq(variables['Purpose'], 'ElevatedPesterTests_xUnit_Packaging') diff --git a/build.psm1 b/build.psm1 index 1d8167a05d..1d0122bd22 100644 --- a/build.psm1 +++ b/build.psm1 @@ -2744,45 +2744,6 @@ function Save-PSOptions { $Options | ConvertTo-Json -Depth 3 | Out-File -Encoding utf8 -FilePath $PSOptionsPath } -# Restore PSOptions -# Optionally remove the PSOptions file -function Restore-PSOptions { - param( - [ValidateScript({Test-Path $_})] - [string] - $PSOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath 'psoptions.json'), - [switch] - $Remove - ) - - $options = Get-Content -Path $PSOptionsPath | ConvertFrom-Json - - if($Remove) - { - # Remove PSOptions. - # The file is only used to set the PSOptions. - Remove-Item -Path $psOptionsPath - } - - Set-PSOptions -Options $options -} - -# Save PSOptions to be restored by Restore-PSOptions -function Save-PSOptions { - param( - [ValidateScript({$parent = Split-Path $_;if($parent){Test-Path $parent}else{return $true}})] - [ValidateNotNullOrEmpty()] - [string] - $PSOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath 'psoptions.json'), - - [ValidateNotNullOrEmpty()] - [object] - $Options = (Get-PSOptions -DefaultToNew) - ) - - $Options | ConvertTo-Json -Depth 3 | Out-File -Encoding utf8 -FilePath $PSOptionsPath -} - # Restore PSOptions # Optionally remove the PSOptions file function Restore-PSOptions { diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 97de79f97e..e67f6926fc 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -191,6 +191,20 @@ function Invoke-AppVeyorBuild } Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -CI -ReleaseTag $releaseTag + + Save-PSOptions + + $options = (Get-PSOptions) + + $path = split-path -path $options.Output + + $psOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath '../psoptions.json') + $buildZipPath = (Join-Path -Path $PSScriptRoot -ChildPath '../build.zip') + + Compress-Archive -Path $path -DestinationPath $buildZipPath + + Push-Artifact -Path $psOptionsPath -Name 'build' + Push-Artifact -Path $buildZipPath -Name 'build' } # Implements the CI 'install' step @@ -263,13 +277,41 @@ function Update-TestResults } } +function Invoke-CIxUnit +{ + param( + [switch] + $SkipFailing + ) + $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions)) + if(!(Test-Path "$env:CoreOutput\pwsh.exe")) + { + throw "CoreCLR pwsh.exe was not built" + } + + $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml" + + Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Write-Host -ForegroundColor Green 'Uploading PSxUnit test results' + Update-TestResults -resultsFile $ParallelXUnitTestResultsFile + Push-Artifact -Path $ParallelXUnitTestResultsFile -name xunit + + if(!$SkipFailing.IsPresent) + { + # Fail the build, if tests failed + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile + } +} + # Implement CI 'Test_script' function Invoke-AppVeyorTest { [CmdletBinding()] param( - [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests_xUnit_Packaging')] - [string] $Purpose + [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests')] + [string] $Purpose, + [ValidateSet('CI', 'Others')] + [string] $TagSet ) # CoreCLR @@ -277,7 +319,6 @@ function Invoke-AppVeyorTest Write-Host -Foreground Green 'Run CoreCLR tests' $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml" $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml" - $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml" if(!(Test-Path "$env:CoreOutput\pwsh.exe")) { throw "CoreCLR pwsh.exe was not built" @@ -286,15 +327,18 @@ function Invoke-AppVeyorTest # Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently # https://github.com/pester/Pester/issues/608 # To work-around it, we exlude all categories, but 'CI' from the list - if (Test-DailyBuild) - { - $ExcludeTag = @() - Write-Host -Foreground Green 'Running all CoreCLR tests..' - } - else - { - $ExcludeTag = @('Slow', 'Feature', 'Scenario') - Write-Host -Foreground Green 'Running "CI" CoreCLR tests..' + switch ($TagSet) { + 'CI' { + Write-Host -Foreground Green 'Running "CI" CoreCLR tests..' + $ExcludeTag = @('Slow', 'Feature', 'Scenario') + } + 'Others' { + Write-Host -Foreground Green 'Running non-CI CoreCLR tests..' + $ExcludeTag = @('CI') + } + Default { + throw "Unknow TagSet: '$TagSet'" + } } # Get the experimental feature names and the tests associated with them @@ -309,7 +353,7 @@ function Invoke-AppVeyorTest Tag = @() ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows' } - Start-PSPester @arguments -Title 'Pester Unelevated' + Start-PSPester @arguments -Title "Pester Unelevated - $TagSet" Write-Host -Foreground Green 'Upload CoreCLR Non-Admin test results' Update-TestResults -resultsFile $testResultsNonAdminFile # Fail the build, if tests failed @@ -341,7 +385,7 @@ function Invoke-AppVeyorTest } } - if ($Purpose -eq 'ElevatedPesterTests_xUnit_Packaging') { + if ($Purpose -eq 'ElevatedPesterTests') { $arguments = @{ Terse = $true Bindir = $env:CoreOutput @@ -349,17 +393,13 @@ function Invoke-AppVeyorTest Tag = @('RequireAdminOnWindows') ExcludeTag = $ExcludeTag } - Start-PSPester @arguments -Title 'Pester Elevated' + Start-PSPester @arguments -Title "Pester Elevated - $TagSet" Write-Host -Foreground Green 'Upload CoreCLR Admin test results' Update-TestResults -resultsFile $testResultsAdminFile - Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile - Write-Host -ForegroundColor Green 'Uploading PSxUnit test results' - Update-TestResults -resultsFile $ParallelXUnitTestResultsFile # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $testResultsAdminFile - Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile # Run tests with specified experimental features enabled foreach ($entry in $ExperimentalFeatureTests.GetEnumerator())