diff --git a/build.psm1 b/build.psm1 index 490505cc08..7e37977742 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1369,7 +1369,6 @@ function Test-PSPesterResults function Start-PSxUnit { [CmdletBinding()]param( - [string] $SequentialTestResultsFile = "SequentialXUnitResults.xml", [string] $ParallelTestResultsFile = "ParallelXUnitResults.xml" ) @@ -1382,10 +1381,9 @@ function Start-PSxUnit { } try { - Push-Location $PSScriptRoot/test/csharp + Push-Location $PSScriptRoot/test/xUnit # Path manipulation to obtain test project output directory - dotnet restore if(-not $Environment.IsWindows) { @@ -1417,30 +1415,22 @@ function Start-PSxUnit { } } - # Run sequential tests first, and then run the tests that can execute in parallel - if (Test-Path $SequentialTestResultsFile) { - Remove-Item $SequentialTestResultsFile -Force -ErrorAction SilentlyContinue - } - dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Sequential -p:ParallelizeTestCollections=false --test-adapter-path:. "--logger:xunit;LogFilePath=$SequentialTestResultsFile" - Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential' + dotnet build --configuration $Options.configuration - $extraParams = @() + if (Test-Path $ParallelTestResultsFile) { + Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue + } # we are having intermittent issues on macOS with these tests failing. # VSTS has suggested forcing them to be sequential if($env:TF_BUILD -and $IsMacOS) { Write-Log 'Forcing parallel xunit tests to run sequentially.' - $extraParams += @( - '-parallel' - 'none' - ) + dotnet test -p:ParallelizeTestCollections=false --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" + } else { + dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" } - if (Test-Path $ParallelTestResultsFile) { - Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue - } - dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Parallel --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel' } finally { diff --git a/test/README.md b/test/README.md index 08e07d7a54..d511995cbe 100644 --- a/test/README.md +++ b/test/README.md @@ -4,8 +4,7 @@ Testing The tests are organized by testing language. Thus Pester tests, which are written in the PowerShell language, are in [./powershell](./powershell) and xUnit tests, written in C#, are in -[./csharp](./csharp). The sanity tests for the Full .NET build of -PowerShell are in [./fullclr](./fullclr), and the third-party +[./xUnit](./xUnit). The third-party [shebang][] test is in [./shebang](./shebang). [shebang]: https://en.wikipedia.org/wiki/Shebang_(Unix) diff --git a/test/csharp/README.md b/test/csharp/README.md deleted file mode 100644 index 18979b0552..0000000000 --- a/test/csharp/README.md +++ /dev/null @@ -1,20 +0,0 @@ -xUnit Tests -=========== - -These tests are completely Linux specific. - -Every test class *must* belong to -`[Collection("AssemblyLoadContext")]`. This ensures that PowerShell's -AssemblyLoadContext is initialized before any other code is executed. -When this is not the case, late initialization fails with -`System.InvalidOperationException : Binding model is already locked -for the AppDomain and cannot be reset.` - -Having every class in the same collection is as close to an xUnit -global init hook as can be done. - -Running xUnit Tests -------------------- - -Go to the top level of the PowerShell repository and run: -`Start-PSxUnit` inside a self-hosted copy of PowerShell. diff --git a/test/xUnit/Asserts/PriorityAttribute.cs b/test/xUnit/Asserts/PriorityAttribute.cs new file mode 100644 index 0000000000..626ac5ac23 --- /dev/null +++ b/test/xUnit/Asserts/PriorityAttribute.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class TestPriorityAttribute : Attribute +{ + public TestPriorityAttribute(int priority) + { + Priority = priority; + } + + public int Priority { get; private set; } +} diff --git a/test/xUnit/Asserts/PriorityOrderer.cs b/test/xUnit/Asserts/PriorityOrderer.cs new file mode 100644 index 0000000000..72eff865a9 --- /dev/null +++ b/test/xUnit/Asserts/PriorityOrderer.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace TestOrder.TestCaseOrdering +{ + public class PriorityOrderer : ITestCaseOrderer + { + public IEnumerable OrderTestCases(IEnumerable testCases) where TTestCase : ITestCase + { + var sortedMethods = new SortedDictionary>(); + + foreach (TTestCase testCase in testCases) + { + int priority = 0; + + foreach (IAttributeInfo attr in testCase.TestMethod.Method.GetCustomAttributes((typeof(TestPriorityAttribute).AssemblyQualifiedName))) + priority = attr.GetNamedArgument("Priority"); + + GetOrCreate(sortedMethods, priority).Add(testCase); + } + + foreach (var list in sortedMethods.Keys.Select(priority => sortedMethods[priority])) + { + list.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.TestMethod.Method.Name, y.TestMethod.Method.Name)); + foreach (TTestCase testCase in list) + yield return testCase; + } + } + + static TValue GetOrCreate(IDictionary dictionary, TKey key) where TValue : new() + { + TValue result; + + if (dictionary.TryGetValue(key, out result)) return result; + + result = new TValue(); + dictionary[key] = result; + + return result; + } + } +} diff --git a/test/xUnit/README.md b/test/xUnit/README.md new file mode 100644 index 0000000000..1debb2abb4 --- /dev/null +++ b/test/xUnit/README.md @@ -0,0 +1,29 @@ +# xUnit Tests + +The folder contains xUnit tests for PowerShell Core project. + +## Running xUnit Tests + +Go to the top level of the PowerShell repository and run full set of tests: +`Start-PSxUnit` inside a self-hosted copy of PowerShell. + +Go to the test project folder and run `dotnet test -c Release`. + +Use [`filter`](xunit-filter) parameter to run only needed tests: +```powershell +dotnet test -c Release --filter "FullyQualifiedName~UnitTest1 # Runs tests which have UnitTest1 in FullyQualifiedName +dotnet test --filter Name~TestMethod1 # Runs tests whose name contains TestMethod1 +``` + +## Creating xUnit Tests + +Keep the folder structure that is for Pester [../../test/powershell](../../test/powershell) and C# files [../../src](../../src). + +Use namespace names started with `PSTests`. +```c# +namespace PSTests.YourNameSpace +{ +} +``` + +[xunit-filter]: https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests \ No newline at end of file diff --git a/test/csharp/test_Binders.cs b/test/xUnit/csharp/test_Binders.cs similarity index 100% rename from test/csharp/test_Binders.cs rename to test/xUnit/csharp/test_Binders.cs diff --git a/test/csharp/test_CorePsPlatform.cs b/test/xUnit/csharp/test_CorePsPlatform.cs similarity index 100% rename from test/csharp/test_CorePsPlatform.cs rename to test/xUnit/csharp/test_CorePsPlatform.cs diff --git a/test/csharp/test_ExtensionMethods.cs b/test/xUnit/csharp/test_ExtensionMethods.cs similarity index 100% rename from test/csharp/test_ExtensionMethods.cs rename to test/xUnit/csharp/test_ExtensionMethods.cs diff --git a/test/csharp/test_FileSystemProvider.cs b/test/xUnit/csharp/test_FileSystemProvider.cs similarity index 100% rename from test/csharp/test_FileSystemProvider.cs rename to test/xUnit/csharp/test_FileSystemProvider.cs diff --git a/test/csharp/test_MshSnapinInfo.cs b/test/xUnit/csharp/test_MshSnapinInfo.cs similarity index 100% rename from test/csharp/test_MshSnapinInfo.cs rename to test/xUnit/csharp/test_MshSnapinInfo.cs diff --git a/test/csharp/test_PSConfiguration.cs b/test/xUnit/csharp/test_PSConfiguration.cs similarity index 99% rename from test/csharp/test_PSConfiguration.cs rename to test/xUnit/csharp/test_PSConfiguration.cs index 173dea6810..af47feee60 100644 --- a/test/csharp/test_PSConfiguration.cs +++ b/test/xUnit/csharp/test_PSConfiguration.cs @@ -12,6 +12,7 @@ using Newtonsoft.Json.Linq; namespace PSTests.Sequential { + [TestCaseOrderer("TestOrder.TestCaseOrdering.PriorityOrderer", "powershell-tests")] public class PowerShellPolicyFixture : IDisposable { private const string configFileName = "powershell.config.json"; @@ -344,7 +345,7 @@ namespace PSTests.Sequential this.fixture = fixture; } - [Fact] + [Fact, TestPriority(1)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotEmpty() { fixture.SetupConfigFile1(); @@ -358,7 +359,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies); } - [Fact] + [Fact, TestPriority(2)] public void PowerShellConfig_GetPowerShellPolicies_EmptyUserConfig() { fixture.SetupConfigFile2(); @@ -371,7 +372,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(sysPolicies, fixture.SystemWidePolicies); } - [Fact] + [Fact, TestPriority(3)] public void PowerShellConfig_GetPowerShellPolicies_EmptySystemConfig() { fixture.SetupConfigFile3(); @@ -384,7 +385,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies); } - [Fact] + [Fact, TestPriority(4)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesEmpty() { fixture.SetupConfigFile4(); @@ -395,7 +396,7 @@ namespace PSTests.Sequential Assert.Null(userPolicies); } - [Fact] + [Fact, TestPriority(5)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotExist() { fixture.CleanupConfigFiles(); @@ -406,7 +407,7 @@ namespace PSTests.Sequential Assert.Null(userPolicies); } - [Fact] + [Fact, TestPriority(6)] public void Utils_GetPolicySetting_BothConfigFilesNotEmpty() { fixture.SetupConfigFile1(); @@ -504,7 +505,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration); } - [Fact] + [Fact, TestPriority(7)] public void Utils_GetPolicySetting_EmptyUserConfig() { fixture.SetupConfigFile2(); @@ -602,7 +603,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration); } - [Fact] + [Fact, TestPriority(8)] public void Utils_GetPolicySetting_EmptySystemConfig() { fixture.SetupConfigFile3(); @@ -701,7 +702,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null); } - [Fact] + [Fact, TestPriority(9)] public void Utils_GetPolicySetting_BothConfigFilesEmpty() { fixture.SetupConfigFile4(); @@ -800,7 +801,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null); } - [Fact] + [Fact, TestPriority(10)] public void Utils_GetPolicySetting_BothConfigFilesNotExist() { fixture.CleanupConfigFiles(); diff --git a/test/csharp/test_PSVersionInfo.cs b/test/xUnit/csharp/test_PSVersionInfo.cs similarity index 100% rename from test/csharp/test_PSVersionInfo.cs rename to test/xUnit/csharp/test_PSVersionInfo.cs diff --git a/test/csharp/test_Runspace.cs b/test/xUnit/csharp/test_Runspace.cs similarity index 97% rename from test/csharp/test_Runspace.cs rename to test/xUnit/csharp/test_Runspace.cs index c9f0611240..1625435466 100644 --- a/test/csharp/test_Runspace.cs +++ b/test/xUnit/csharp/test_Runspace.cs @@ -66,7 +66,8 @@ namespace PSTests.Parallel [Fact] public void TestRunspaceWithPowerShellAndInitialSessionState() { - InitialSessionState iss = InitialSessionState.CreateDefault2(); + // CreateDefault2 is intentional. + InitialSessionState iss = InitialSessionState.CreateDefault(); // NOTE: instantiate custom host myHost for the next line to capture stdout and stderr output // in addition to just the PSObjects @@ -90,9 +91,11 @@ namespace PSTests.Parallel ++objCount; Assert.NotNull(result); } + Assert.Equal(count, objCount); - powerShell.Dispose(); } + + runspace.Close(); } } } diff --git a/test/csharp/test_SecuritySupport.cs b/test/xUnit/csharp/test_SecuritySupport.cs similarity index 100% rename from test/csharp/test_SecuritySupport.cs rename to test/xUnit/csharp/test_SecuritySupport.cs diff --git a/test/csharp/test_SessionState.cs b/test/xUnit/csharp/test_SessionState.cs similarity index 100% rename from test/csharp/test_SessionState.cs rename to test/xUnit/csharp/test_SessionState.cs diff --git a/test/csharp/test_Utils.cs b/test/xUnit/csharp/test_Utils.cs similarity index 100% rename from test/csharp/test_Utils.cs rename to test/xUnit/csharp/test_Utils.cs diff --git a/test/csharp/csharp.tests.csproj b/test/xUnit/xUnit.tests.csproj similarity index 95% rename from test/csharp/csharp.tests.csproj rename to test/xUnit/xUnit.tests.csproj index af62b5447e..72f5259391 100644 --- a/test/csharp/csharp.tests.csproj +++ b/test/xUnit/xUnit.tests.csproj @@ -5,6 +5,7 @@ PowerShell xUnit Tests powershell-tests + true win7-x86;win7-x64;osx-x64;linux-x64 @@ -26,5 +27,4 @@ - diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 34164db5fc..cda875998a 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -369,7 +369,6 @@ function Invoke-AppVeyorTest Write-Host -Foreground Green 'Run CoreCLR tests' $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml" $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml" - $SequentialXUnitTestResultsFile = "$pwd\SequentialXUnitTestResults.xml" $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml" if(!(Test-Path "$env:CoreOutput\pwsh.exe")) { @@ -443,19 +442,13 @@ function Invoke-AppVeyorTest Write-Host -Foreground Green 'Upload CoreCLR Admin test results' Update-AppVeyorTestResults -resultsFile $testResultsAdminFile - Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile Write-Host -ForegroundColor Green 'Uploading PSxUnit test results' - Update-AppVeyorTestResults -resultsFile $SequentialXUnitTestResultsFile Update-AppVeyorTestResults -resultsFile $ParallelXUnitTestResultsFile # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $testResultsAdminFile - @( - $SequentialXUnitTestResultsFile, - $ParallelXUnitTestResultsFile - ) | ForEach-Object { - Test-XUnitTestResults -TestResultsFile $_ - } + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile # Run tests with specified experimental features enabled foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { diff --git a/tools/travis.ps1 b/tools/travis.ps1 index a9823e2a6c..33498d89ec 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -330,12 +330,11 @@ elseif($Stage -eq 'Build') } try { - $SequentialXUnitTestResultsFile = "$pwd/SequentialXUnitTestResults.xml" $ParallelXUnitTestResultsFile = "$pwd/ParallelXUnitTestResults.xml" - Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile # If there are failures, Test-XUnitTestResults throws - $SequentialXUnitTestResultsFile, $ParallelXUnitTestResultsFile | ForEach-Object { Test-XUnitTestResults -TestResultsFile $_ } + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile } catch { $result = "FAIL"