From 25c127c3bc52c13bed45a03241644d8c455d4f2a Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 3 Aug 2018 11:20:14 -0700 Subject: [PATCH] Update CI scripts to support running tests for experimental features (#7419) --- build.psm1 | 81 +++++++++++++++++++++++++++++------- test/tools/TestMetadata.json | 5 +++ tools/appveyor.psm1 | 73 ++++++++++++++++++++++++++++++-- tools/travis.ps1 | 79 ++++++++++++++++++++++++++++------- 4 files changed, 203 insertions(+), 35 deletions(-) create mode 100644 test/tools/TestMetadata.json diff --git a/build.psm1 b/build.psm1 index e590387da6..44ddfa639d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -973,6 +973,18 @@ function Publish-PSTestTools { } } +function Get-ExperimentalFeatureTests { + $testMetadataFile = Join-Path $PSScriptRoot "test/tools/TestMetadata.json" + $metadata = Get-Content -Path $testMetadataFile -Raw | ConvertFrom-Json | ForEach-Object -MemberName ExperimentalFeatures + $features = $metadata | Get-Member -MemberType NoteProperty | ForEach-Object -MemberName Name + + $featureTests = @{} + foreach ($featureName in $features) { + $featureTests[$featureName] = $metadata.$featureName + } + $featureTests +} + function Start-PSPester { [CmdletBinding(DefaultParameterSetName='default')] param( @@ -983,9 +995,9 @@ function Start-PSPester { [string[]]$ExcludeTag = 'Slow', [string[]]$Tag = @("CI","Feature"), [switch]$ThrowOnFailure, - [string]$binDir = (Split-Path (Get-PSOptions -DefaultToNew).Output), - [string]$powershell = (Join-Path $binDir 'pwsh'), - [string]$Pester = ([IO.Path]::Combine($binDir, "Modules", "Pester")), + [string]$BinDir = (Split-Path (Get-PSOptions -DefaultToNew).Output), + [string]$powershell = (Join-Path $BinDir 'pwsh'), + [string]$Pester = ([IO.Path]::Combine($BinDir, "Modules", "Pester")), [Parameter(ParameterSetName='Unelevate',Mandatory=$true)] [switch]$Unelevate, [switch]$Quiet, @@ -994,7 +1006,8 @@ function Start-PSPester { [switch]$PassThru, [Parameter(ParameterSetName='PassThru',HelpMessage='Run commands on Linux with sudo.')] [switch]$Sudo, - [switch]$IncludeFailingTest + [switch]$IncludeFailingTest, + [string]$ExperimentalFeatureName ) if (-not (Get-Module -ListAvailable -Name $Pester -ErrorAction SilentlyContinue | Where-Object { $_.Version -ge "4.2" } )) @@ -1151,6 +1164,37 @@ function Start-PSPester { } } + $PSFlags = @("-noprofile") + if (-not [string]::IsNullOrEmpty($ExperimentalFeatureName)) { + $configFile = [System.IO.Path]::GetTempFileName() + $configFile = [System.IO.Path]::ChangeExtension($configFile, ".json") + + ## Create the config.json file to enable the given experimental feature. + ## On Windows, we need to have 'RemoteSigned' declared for ExecutionPolicy because the ExecutionPolicy is 'Restricted' by default. + ## On Unix, ExecutionPolicy is not supported, so we don't need to declare it. + if ($Environment.IsWindows) { + $content = @" +{ + "Microsoft.PowerShell:ExecutionPolicy":"RemoteSigned", + "ExperimentalFeatures": [ + "$ExperimentalFeatureName" + ] +} +"@ + } else { + $content = @" +{ + "ExperimentalFeatures": [ + "$ExperimentalFeatureName" + ] +} +"@ + } + + Set-Content -Path $configFile -Value $content -Encoding Ascii -Force + $PSFlags = @("-settings", $configFile, "-noprofile") + } + # To ensure proper testing, the module path must not be inherited by the spawned process try { $originalModulePath = $env:PSModulePath @@ -1158,7 +1202,7 @@ function Start-PSPester { $env:POWERSHELL_TELEMETRY_OPTOUT = 1 if ($Unelevate) { - Start-UnelevatedProcess -process $powershell -arguments @('-noprofile', '-c', $Command) + Start-UnelevatedProcess -process $powershell -arguments ($PSFlags + "-c $Command") $currentLines = 0 while ($true) { @@ -1197,11 +1241,11 @@ function Start-PSPester { $passThruFile = [System.IO.Path]::GetTempFileName() try { - $command += "|Export-Clixml -Path '$passThruFile' -Force" + $command += "| Export-Clixml -Path '$passThruFile' -Force" - $passThruCommand = {& $powershell -noprofile -c $command } + $passThruCommand = { & $powershell $PSFlags -c $command } if ($Sudo.IsPresent) { - $passThruCommand = {& sudo $powershell -noprofile -c $command } + $passThruCommand = { & sudo $powershell $PSFlags -c $command } } $writeCommand = { Write-Host $_ } @@ -1222,11 +1266,11 @@ function Start-PSPester { { if ($Terse) { - Start-NativeExecution -sb {& $powershell -noprofile -c $command} | ForEach-Object { Write-Terse -line $_ } + Start-NativeExecution -sb {& $powershell $PSFlags -c $command} | ForEach-Object { Write-Terse -line $_ } } else { - Start-NativeExecution -sb {& $powershell -noprofile -c $command} + Start-NativeExecution -sb {& $powershell $PSFlags -c $command} } } } @@ -1352,12 +1396,17 @@ function Test-PSPesterResults [CmdletBinding(DefaultParameterSetName='file')] param( [Parameter(ParameterSetName='file')] - [string]$TestResultsFile = "pester-tests.xml", + [string] $TestResultsFile = "pester-tests.xml", + [Parameter(ParameterSetName='file')] - [string]$TestArea = 'test/powershell', - [Parameter(ParameterSetName='PesterPassThruObject',Mandatory)] - [pscustomobject] $ResultObject - ) + [string] $TestArea = 'test/powershell', + + [Parameter(ParameterSetName='PesterPassThruObject', Mandatory)] + [pscustomobject] $ResultObject, + + [Parameter(ParameterSetName='PesterPassThruObject')] + [switch] $CanHaveNoResult + ) if($PSCmdLet.ParameterSetName -eq 'file') { @@ -1388,7 +1437,7 @@ function Test-PSPesterResults } elseif ($PSCmdLet.ParameterSetName -eq 'PesterPassThruObject') { - if ($ResultObject.TotalCount -le 0) + if ($ResultObject.TotalCount -le 0 -and -not $CanHaveNoResult) { throw 'NO TESTS RUN' } diff --git a/test/tools/TestMetadata.json b/test/tools/TestMetadata.json new file mode 100644 index 0000000000..fbfcbcf45c --- /dev/null +++ b/test/tools/TestMetadata.json @@ -0,0 +1,5 @@ +{ + "ExperimentalFeatures": { + "ExpTest.FeatureOne": [ "test/powershell/engine/ExperimentalFeature" ] + } +} diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 4299554577..c6ff21c695 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -339,26 +339,67 @@ function Invoke-AppVeyorTest # Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently # https://github.com/pester/Pester/issues/608 # To work-around it, we exlude all categories, but 'CI' from the list - $ExcludeTag = @('Slow', 'Feature', 'Scenario') if (Test-DailyBuild) { $ExcludeTag = @() Write-Host -Foreground Green 'Running all CoreCLR tests..' } else { + $ExcludeTag = @('Slow', 'Feature', 'Scenario') Write-Host -Foreground Green 'Running "CI" CoreCLR tests..' } + # Get the experimental feature names and the tests associated with them + $ExperimentalFeatureTests = Get-ExperimentalFeatureTests + if ($Purpose -eq 'UnelevatedPesterTests') { - Start-PSPester -Terse -bindir $env:CoreOutput -outputFile $testResultsNonAdminFile -Unelevate -Tag @() -ExcludeTag ($ExcludeTag + @('RequireAdminOnWindows')) + $arguments = @{ + Bindir = $env:CoreOutput + OutputFile = $testResultsNonAdminFile + Unelevate = $true + Terse = $true + Tag = @() + ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows' + } + Start-PSPester @arguments Write-Host -Foreground Green 'Upload CoreCLR Non-Admin test results' Update-AppVeyorTestResults -resultsFile $testResultsNonAdminFile - # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $testResultsNonAdminFile + + # Run tests with specified experimental features enabled + foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { + $featureName = $entry.Key + $testFiles = $entry.Value + + $expFeatureTestResultFile = "$pwd\TestsResultsNonAdmin.$featureName.xml" + $arguments['OutputFile'] = $expFeatureTestResultFile + $arguments['ExperimentalFeatureName'] = $featureName + if ($testFiles.Count -eq 0) { + # If an empty array is specified for the feature name, we run all tests with the feature enabled. + # This allows us to prevent regressions to a critical engine experimental feature. + $arguments.Remove('Path') + } else { + # If a non-empty string or array is specified for the feature name, we only run those test files. + $arguments['Path'] = $testFiles + } + Start-PSPester @arguments + + Write-Host -ForegroundColor Green "Upload CoreCLR Non-Admin test results for experimental feature '$featureName'" + Update-AppVeyorTestResults -resultsFile $expFeatureTestResultFile + # Fail the build, if tests failed + Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile + } } if ($Purpose -eq 'ElevatedPesterTests_xUnit_Packaging') { - Start-PSPester -Terse -bindir $env:CoreOutput -outputFile $testResultsAdminFile -Tag @('RequireAdminOnWindows') -ExcludeTag $ExcludeTag + $arguments = @{ + Terse = $true + Bindir = $env:CoreOutput + OutputFile = $testResultsAdminFile + Tag = @('RequireAdminOnWindows') + ExcludeTag = $ExcludeTag + } + Start-PSPester @arguments Write-Host -Foreground Green 'Upload CoreCLR Admin test results' Update-AppVeyorTestResults -resultsFile $testResultsAdminFile @@ -375,6 +416,30 @@ function Invoke-AppVeyorTest ) | ForEach-Object { Test-XUnitTestResults -TestResultsFile $_ } + + # Run tests with specified experimental features enabled + foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { + $featureName = $entry.Key + $testFiles = $entry.Value + + $expFeatureTestResultFile = "$pwd\TestsResultsAdmin.$featureName.xml" + $arguments['OutputFile'] = $expFeatureTestResultFile + $arguments['ExperimentalFeatureName'] = $featureName + if ($testFiles.Count -eq 0) { + # If an empty array is specified for the feature name, we run all tests with the feature enabled. + # This allows us to prevent regressions to a critical engine experimental feature. + $arguments.Remove('Path') + } else { + # If a non-empty string or array is specified for the feature name, we only run those test files. + $arguments['Path'] = $testFiles + } + Start-PSPester @arguments + + Write-Host -ForegroundColor Green "Upload CoreCLR Admin test results for experimental feature '$featureName'" + Update-AppVeyorTestResults -resultsFile $expFeatureTestResultFile + # Fail the build, if tests failed + Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile + } } Set-BuildVariable -Name TestPassed -Value True diff --git a/tools/travis.ps1 b/tools/travis.ps1 index bafd66771f..ef43a8c8a2 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -204,8 +204,8 @@ elseif($Stage -eq 'Build') $testResultsNoSudo = "$pwd/TestResultsNoSudo.xml" $testResultsSudo = "$pwd/TestResultsSudo.xml" - $pesterParam = @{ - 'binDir' = $output + $noSudoPesterParam = @{ + 'BinDir' = $output 'PassThru' = $true 'Terse' = $true 'Tag' = @() @@ -214,31 +214,80 @@ elseif($Stage -eq 'Build') } if ($isFullBuild) { - $pesterParam['Tag'] = @('CI','Feature','Scenario') + $noSudoPesterParam['Tag'] = @('CI','Feature','Scenario') } else { - $pesterParam['Tag'] = @('CI') - $pesterParam['ThrowOnFailure'] = $true + $noSudoPesterParam['Tag'] = @('CI') + $noSudoPesterParam['ThrowOnFailure'] = $true } - if ($hasRunFailingTestTag) - { - $pesterParam['IncludeFailingTest'] = $true + if ($hasRunFailingTestTag) { + $noSudoPesterParam['IncludeFailingTest'] = $true } + # Get the experimental feature names and the tests associated with them + $ExperimentalFeatureTests = Get-ExperimentalFeatureTests + # Running tests which do not require sudo. - $pesterPassThruNoSudoObject = Start-PSPester @pesterParam + $pesterPassThruNoSudoObject = Start-PSPester @noSudoPesterParam + + # Running tests that do not require sudo, with specified experimental features enabled + $noSudoResultsWithExpFeatures = @() + foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { + $featureName = $entry.Key + $testFiles = $entry.Value + + $expFeatureTestResultFile = "$pwd\TestResultsNoSudo.$featureName.xml" + $noSudoPesterParam['OutputFile'] = $expFeatureTestResultFile + $noSudoPesterParam['ExperimentalFeatureName'] = $featureName + if ($testFiles.Count -eq 0) { + # If an empty array is specified for the feature name, we run all tests with the feature enabled. + # This allows us to prevent regressions to a critical engine experimental feature. + $noSudoPesterParam.Remove('Path') + } else { + # If a non-empty string or array is specified for the feature name, we only run those test files. + $noSudoPesterParam['Path'] = $testFiles + } + $passThruResult = Start-PSPester @noSudoPesterParam + $noSudoResultsWithExpFeatures += $passThruResult + } # Running tests, which require sudo. - $pesterParam['Tag'] = @('RequireSudoOnUnix') - $pesterParam['ExcludeTag'] = @() - $pesterParam['Sudo'] = $true - $pesterParam['OutputFile'] = $testResultsSudo - $pesterPassThruSudoObject = Start-PSPester @pesterParam + $sudoPesterParam = $noSudoPesterParam.Clone() + $sudoPesterParam.Remove('Path') + $sudoPesterParam['Tag'] = @('RequireSudoOnUnix') + $sudoPesterParam['ExcludeTag'] = @() + $sudoPesterParam['Sudo'] = $true + $sudoPesterParam['OutputFile'] = $testResultsSudo + $pesterPassThruSudoObject = Start-PSPester @sudoPesterParam + + # Running tests that require sudo, with specified experimental features enabled + $sudoResultsWithExpFeatures = @() + foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { + $featureName = $entry.Key + $testFiles = $entry.Value + + $expFeatureTestResultFile = "$pwd\TestResultsSudo.$featureName.xml" + $sudoPesterParam['OutputFile'] = $expFeatureTestResultFile + $sudoPesterParam['ExperimentalFeatureName'] = $featureName + if ($testFiles.Count -eq 0) { + # If an empty array is specified for the feature name, we run all tests with the feature enabled. + # This allows us to prevent regressions to a critical engine experimental feature. + $sudoPesterParam.Remove('Path') + } else { + # If a non-empty string or array is specified for the feature name, we only run those test files. + $sudoPesterParam['Path'] = $testFiles + } + $passThruResult = Start-PSPester @sudoPesterParam + $sudoResultsWithExpFeatures += $passThruResult + } # Determine whether the build passed try { + $allTestResultsWithNoExpFeature = @($pesterPassThruNoSudoObject, $pesterPassThruSudoObject) + $allTestResultsWithExpFeatures = $noSudoResultsWithExpFeatures + $sudoResultsWithExpFeatures # this throws if there was an error - @($pesterPassThruNoSudoObject, $pesterPassThruSudoObject) | ForEach-Object { Test-PSPesterResults -ResultObject $_ } + $allTestResultsWithNoExpFeature | ForEach-Object { Test-PSPesterResults -ResultObject $_ } + $allTestResultsWithExpFeatures | ForEach-Object { Test-PSPesterResults -ResultObject $_ -CanHaveNoResult } $result = "PASS" } catch {