From a9f106da58b251a5fe92b31903ce74528c3099a2 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 14 Dec 2018 03:22:07 +0500 Subject: [PATCH] Move xUnit tests in new folder (#8356) ## Motivation I have a PR where there are many new xUnit tests. It would also be useful to create new xUnit tests for public APIs. The number of xUnit tests will increase and their ordering is required. ## PR Summary - Move C# xUnit tests in new folder. This allows to put new xUnit tests in directory structure in accordance with directory structure where cs files are. - Use an xUnit TestCaseOrderer attribute to sequentially process tests for `powershell.config.json`. - Update README.md - A race condition was fixed which allowed to run all XUnit tests in single batch job. --- build.psm1 | 26 ++++------ test/README.md | 3 +- test/csharp/README.md | 20 -------- test/xUnit/Asserts/PriorityAttribute.cs | 15 ++++++ test/xUnit/Asserts/PriorityOrderer.cs | 48 +++++++++++++++++++ test/xUnit/README.md | 29 +++++++++++ test/{ => xUnit}/csharp/test_Binders.cs | 0 .../{ => xUnit}/csharp/test_CorePsPlatform.cs | 0 .../csharp/test_ExtensionMethods.cs | 0 .../csharp/test_FileSystemProvider.cs | 0 test/{ => xUnit}/csharp/test_MshSnapinInfo.cs | 0 .../csharp/test_PSConfiguration.cs | 21 ++++---- test/{ => xUnit}/csharp/test_PSVersionInfo.cs | 0 test/{ => xUnit}/csharp/test_Runspace.cs | 7 ++- .../csharp/test_SecuritySupport.cs | 0 test/{ => xUnit}/csharp/test_SessionState.cs | 0 test/{ => xUnit}/csharp/test_Utils.cs | 0 .../xUnit.tests.csproj} | 2 +- tools/appveyor.psm1 | 11 +---- tools/travis.ps1 | 5 +- 20 files changed, 122 insertions(+), 65 deletions(-) delete mode 100644 test/csharp/README.md create mode 100644 test/xUnit/Asserts/PriorityAttribute.cs create mode 100644 test/xUnit/Asserts/PriorityOrderer.cs create mode 100644 test/xUnit/README.md rename test/{ => xUnit}/csharp/test_Binders.cs (100%) rename test/{ => xUnit}/csharp/test_CorePsPlatform.cs (100%) rename test/{ => xUnit}/csharp/test_ExtensionMethods.cs (100%) rename test/{ => xUnit}/csharp/test_FileSystemProvider.cs (100%) rename test/{ => xUnit}/csharp/test_MshSnapinInfo.cs (100%) rename test/{ => xUnit}/csharp/test_PSConfiguration.cs (99%) rename test/{ => xUnit}/csharp/test_PSVersionInfo.cs (100%) rename test/{ => xUnit}/csharp/test_Runspace.cs (97%) rename test/{ => xUnit}/csharp/test_SecuritySupport.cs (100%) rename test/{ => xUnit}/csharp/test_SessionState.cs (100%) rename test/{ => xUnit}/csharp/test_Utils.cs (100%) rename test/{csharp/csharp.tests.csproj => xUnit/xUnit.tests.csproj} (95%) diff --git a/build.psm1 b/build.psm1 index 490505cc08..7e37977742 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1369,7 +1369,6 @@ function Test-PSPesterResults function Start-PSxUnit { [CmdletBinding()]param( - [string] $SequentialTestResultsFile = "SequentialXUnitResults.xml", [string] $ParallelTestResultsFile = "ParallelXUnitResults.xml" ) @@ -1382,10 +1381,9 @@ function Start-PSxUnit { } try { - Push-Location $PSScriptRoot/test/csharp + Push-Location $PSScriptRoot/test/xUnit # Path manipulation to obtain test project output directory - dotnet restore if(-not $Environment.IsWindows) { @@ -1417,30 +1415,22 @@ function Start-PSxUnit { } } - # Run sequential tests first, and then run the tests that can execute in parallel - if (Test-Path $SequentialTestResultsFile) { - Remove-Item $SequentialTestResultsFile -Force -ErrorAction SilentlyContinue - } - dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Sequential -p:ParallelizeTestCollections=false --test-adapter-path:. "--logger:xunit;LogFilePath=$SequentialTestResultsFile" - Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential' + dotnet build --configuration $Options.configuration - $extraParams = @() + if (Test-Path $ParallelTestResultsFile) { + Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue + } # we are having intermittent issues on macOS with these tests failing. # VSTS has suggested forcing them to be sequential if($env:TF_BUILD -and $IsMacOS) { Write-Log 'Forcing parallel xunit tests to run sequentially.' - $extraParams += @( - '-parallel' - 'none' - ) + dotnet test -p:ParallelizeTestCollections=false --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" + } else { + dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" } - if (Test-Path $ParallelTestResultsFile) { - Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue - } - dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Parallel --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel' } finally { diff --git a/test/README.md b/test/README.md index 08e07d7a54..d511995cbe 100644 --- a/test/README.md +++ b/test/README.md @@ -4,8 +4,7 @@ Testing The tests are organized by testing language. Thus Pester tests, which are written in the PowerShell language, are in [./powershell](./powershell) and xUnit tests, written in C#, are in -[./csharp](./csharp). The sanity tests for the Full .NET build of -PowerShell are in [./fullclr](./fullclr), and the third-party +[./xUnit](./xUnit). The third-party [shebang][] test is in [./shebang](./shebang). [shebang]: https://en.wikipedia.org/wiki/Shebang_(Unix) diff --git a/test/csharp/README.md b/test/csharp/README.md deleted file mode 100644 index 18979b0552..0000000000 --- a/test/csharp/README.md +++ /dev/null @@ -1,20 +0,0 @@ -xUnit Tests -=========== - -These tests are completely Linux specific. - -Every test class *must* belong to -`[Collection("AssemblyLoadContext")]`. This ensures that PowerShell's -AssemblyLoadContext is initialized before any other code is executed. -When this is not the case, late initialization fails with -`System.InvalidOperationException : Binding model is already locked -for the AppDomain and cannot be reset.` - -Having every class in the same collection is as close to an xUnit -global init hook as can be done. - -Running xUnit Tests -------------------- - -Go to the top level of the PowerShell repository and run: -`Start-PSxUnit` inside a self-hosted copy of PowerShell. diff --git a/test/xUnit/Asserts/PriorityAttribute.cs b/test/xUnit/Asserts/PriorityAttribute.cs new file mode 100644 index 0000000000..626ac5ac23 --- /dev/null +++ b/test/xUnit/Asserts/PriorityAttribute.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class TestPriorityAttribute : Attribute +{ + public TestPriorityAttribute(int priority) + { + Priority = priority; + } + + public int Priority { get; private set; } +} diff --git a/test/xUnit/Asserts/PriorityOrderer.cs b/test/xUnit/Asserts/PriorityOrderer.cs new file mode 100644 index 0000000000..72eff865a9 --- /dev/null +++ b/test/xUnit/Asserts/PriorityOrderer.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace TestOrder.TestCaseOrdering +{ + public class PriorityOrderer : ITestCaseOrderer + { + public IEnumerable OrderTestCases(IEnumerable testCases) where TTestCase : ITestCase + { + var sortedMethods = new SortedDictionary>(); + + foreach (TTestCase testCase in testCases) + { + int priority = 0; + + foreach (IAttributeInfo attr in testCase.TestMethod.Method.GetCustomAttributes((typeof(TestPriorityAttribute).AssemblyQualifiedName))) + priority = attr.GetNamedArgument("Priority"); + + GetOrCreate(sortedMethods, priority).Add(testCase); + } + + foreach (var list in sortedMethods.Keys.Select(priority => sortedMethods[priority])) + { + list.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.TestMethod.Method.Name, y.TestMethod.Method.Name)); + foreach (TTestCase testCase in list) + yield return testCase; + } + } + + static TValue GetOrCreate(IDictionary dictionary, TKey key) where TValue : new() + { + TValue result; + + if (dictionary.TryGetValue(key, out result)) return result; + + result = new TValue(); + dictionary[key] = result; + + return result; + } + } +} diff --git a/test/xUnit/README.md b/test/xUnit/README.md new file mode 100644 index 0000000000..1debb2abb4 --- /dev/null +++ b/test/xUnit/README.md @@ -0,0 +1,29 @@ +# xUnit Tests + +The folder contains xUnit tests for PowerShell Core project. + +## Running xUnit Tests + +Go to the top level of the PowerShell repository and run full set of tests: +`Start-PSxUnit` inside a self-hosted copy of PowerShell. + +Go to the test project folder and run `dotnet test -c Release`. + +Use [`filter`](xunit-filter) parameter to run only needed tests: +```powershell +dotnet test -c Release --filter "FullyQualifiedName~UnitTest1 # Runs tests which have UnitTest1 in FullyQualifiedName +dotnet test --filter Name~TestMethod1 # Runs tests whose name contains TestMethod1 +``` + +## Creating xUnit Tests + +Keep the folder structure that is for Pester [../../test/powershell](../../test/powershell) and C# files [../../src](../../src). + +Use namespace names started with `PSTests`. +```c# +namespace PSTests.YourNameSpace +{ +} +``` + +[xunit-filter]: https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests \ No newline at end of file diff --git a/test/csharp/test_Binders.cs b/test/xUnit/csharp/test_Binders.cs similarity index 100% rename from test/csharp/test_Binders.cs rename to test/xUnit/csharp/test_Binders.cs diff --git a/test/csharp/test_CorePsPlatform.cs b/test/xUnit/csharp/test_CorePsPlatform.cs similarity index 100% rename from test/csharp/test_CorePsPlatform.cs rename to test/xUnit/csharp/test_CorePsPlatform.cs diff --git a/test/csharp/test_ExtensionMethods.cs b/test/xUnit/csharp/test_ExtensionMethods.cs similarity index 100% rename from test/csharp/test_ExtensionMethods.cs rename to test/xUnit/csharp/test_ExtensionMethods.cs diff --git a/test/csharp/test_FileSystemProvider.cs b/test/xUnit/csharp/test_FileSystemProvider.cs similarity index 100% rename from test/csharp/test_FileSystemProvider.cs rename to test/xUnit/csharp/test_FileSystemProvider.cs diff --git a/test/csharp/test_MshSnapinInfo.cs b/test/xUnit/csharp/test_MshSnapinInfo.cs similarity index 100% rename from test/csharp/test_MshSnapinInfo.cs rename to test/xUnit/csharp/test_MshSnapinInfo.cs diff --git a/test/csharp/test_PSConfiguration.cs b/test/xUnit/csharp/test_PSConfiguration.cs similarity index 99% rename from test/csharp/test_PSConfiguration.cs rename to test/xUnit/csharp/test_PSConfiguration.cs index 173dea6810..af47feee60 100644 --- a/test/csharp/test_PSConfiguration.cs +++ b/test/xUnit/csharp/test_PSConfiguration.cs @@ -12,6 +12,7 @@ using Newtonsoft.Json.Linq; namespace PSTests.Sequential { + [TestCaseOrderer("TestOrder.TestCaseOrdering.PriorityOrderer", "powershell-tests")] public class PowerShellPolicyFixture : IDisposable { private const string configFileName = "powershell.config.json"; @@ -344,7 +345,7 @@ namespace PSTests.Sequential this.fixture = fixture; } - [Fact] + [Fact, TestPriority(1)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotEmpty() { fixture.SetupConfigFile1(); @@ -358,7 +359,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies); } - [Fact] + [Fact, TestPriority(2)] public void PowerShellConfig_GetPowerShellPolicies_EmptyUserConfig() { fixture.SetupConfigFile2(); @@ -371,7 +372,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(sysPolicies, fixture.SystemWidePolicies); } - [Fact] + [Fact, TestPriority(3)] public void PowerShellConfig_GetPowerShellPolicies_EmptySystemConfig() { fixture.SetupConfigFile3(); @@ -384,7 +385,7 @@ namespace PSTests.Sequential fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies); } - [Fact] + [Fact, TestPriority(4)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesEmpty() { fixture.SetupConfigFile4(); @@ -395,7 +396,7 @@ namespace PSTests.Sequential Assert.Null(userPolicies); } - [Fact] + [Fact, TestPriority(5)] public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotExist() { fixture.CleanupConfigFiles(); @@ -406,7 +407,7 @@ namespace PSTests.Sequential Assert.Null(userPolicies); } - [Fact] + [Fact, TestPriority(6)] public void Utils_GetPolicySetting_BothConfigFilesNotEmpty() { fixture.SetupConfigFile1(); @@ -504,7 +505,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration); } - [Fact] + [Fact, TestPriority(7)] public void Utils_GetPolicySetting_EmptyUserConfig() { fixture.SetupConfigFile2(); @@ -602,7 +603,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration); } - [Fact] + [Fact, TestPriority(8)] public void Utils_GetPolicySetting_EmptySystemConfig() { fixture.SetupConfigFile3(); @@ -701,7 +702,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null); } - [Fact] + [Fact, TestPriority(9)] public void Utils_GetPolicySetting_BothConfigFilesEmpty() { fixture.SetupConfigFile4(); @@ -800,7 +801,7 @@ namespace PSTests.Sequential fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null); } - [Fact] + [Fact, TestPriority(10)] public void Utils_GetPolicySetting_BothConfigFilesNotExist() { fixture.CleanupConfigFiles(); diff --git a/test/csharp/test_PSVersionInfo.cs b/test/xUnit/csharp/test_PSVersionInfo.cs similarity index 100% rename from test/csharp/test_PSVersionInfo.cs rename to test/xUnit/csharp/test_PSVersionInfo.cs diff --git a/test/csharp/test_Runspace.cs b/test/xUnit/csharp/test_Runspace.cs similarity index 97% rename from test/csharp/test_Runspace.cs rename to test/xUnit/csharp/test_Runspace.cs index c9f0611240..1625435466 100644 --- a/test/csharp/test_Runspace.cs +++ b/test/xUnit/csharp/test_Runspace.cs @@ -66,7 +66,8 @@ namespace PSTests.Parallel [Fact] public void TestRunspaceWithPowerShellAndInitialSessionState() { - InitialSessionState iss = InitialSessionState.CreateDefault2(); + // CreateDefault2 is intentional. + InitialSessionState iss = InitialSessionState.CreateDefault(); // NOTE: instantiate custom host myHost for the next line to capture stdout and stderr output // in addition to just the PSObjects @@ -90,9 +91,11 @@ namespace PSTests.Parallel ++objCount; Assert.NotNull(result); } + Assert.Equal(count, objCount); - powerShell.Dispose(); } + + runspace.Close(); } } } diff --git a/test/csharp/test_SecuritySupport.cs b/test/xUnit/csharp/test_SecuritySupport.cs similarity index 100% rename from test/csharp/test_SecuritySupport.cs rename to test/xUnit/csharp/test_SecuritySupport.cs diff --git a/test/csharp/test_SessionState.cs b/test/xUnit/csharp/test_SessionState.cs similarity index 100% rename from test/csharp/test_SessionState.cs rename to test/xUnit/csharp/test_SessionState.cs diff --git a/test/csharp/test_Utils.cs b/test/xUnit/csharp/test_Utils.cs similarity index 100% rename from test/csharp/test_Utils.cs rename to test/xUnit/csharp/test_Utils.cs diff --git a/test/csharp/csharp.tests.csproj b/test/xUnit/xUnit.tests.csproj similarity index 95% rename from test/csharp/csharp.tests.csproj rename to test/xUnit/xUnit.tests.csproj index af62b5447e..72f5259391 100644 --- a/test/csharp/csharp.tests.csproj +++ b/test/xUnit/xUnit.tests.csproj @@ -5,6 +5,7 @@ PowerShell xUnit Tests powershell-tests + true win7-x86;win7-x64;osx-x64;linux-x64 @@ -26,5 +27,4 @@ - diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 34164db5fc..cda875998a 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -369,7 +369,6 @@ function Invoke-AppVeyorTest Write-Host -Foreground Green 'Run CoreCLR tests' $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml" $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml" - $SequentialXUnitTestResultsFile = "$pwd\SequentialXUnitTestResults.xml" $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml" if(!(Test-Path "$env:CoreOutput\pwsh.exe")) { @@ -443,19 +442,13 @@ function Invoke-AppVeyorTest Write-Host -Foreground Green 'Upload CoreCLR Admin test results' Update-AppVeyorTestResults -resultsFile $testResultsAdminFile - Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile Write-Host -ForegroundColor Green 'Uploading PSxUnit test results' - Update-AppVeyorTestResults -resultsFile $SequentialXUnitTestResultsFile Update-AppVeyorTestResults -resultsFile $ParallelXUnitTestResultsFile # Fail the build, if tests failed Test-PSPesterResults -TestResultsFile $testResultsAdminFile - @( - $SequentialXUnitTestResultsFile, - $ParallelXUnitTestResultsFile - ) | ForEach-Object { - Test-XUnitTestResults -TestResultsFile $_ - } + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile # Run tests with specified experimental features enabled foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) { diff --git a/tools/travis.ps1 b/tools/travis.ps1 index a9823e2a6c..33498d89ec 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -330,12 +330,11 @@ elseif($Stage -eq 'Build') } try { - $SequentialXUnitTestResultsFile = "$pwd/SequentialXUnitTestResults.xml" $ParallelXUnitTestResultsFile = "$pwd/ParallelXUnitTestResults.xml" - Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile # If there are failures, Test-XUnitTestResults throws - $SequentialXUnitTestResultsFile, $ParallelXUnitTestResultsFile | ForEach-Object { Test-XUnitTestResults -TestResultsFile $_ } + Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile } catch { $result = "FAIL"