From b3d0913d74d202a9b58fa40ac03d363a82152d9d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 10 Aug 2018 15:13:31 -0700 Subject: [PATCH] Add CI definition for VSTS for macOS (#7490) Add CI definition for VSTS for macOS - Add VSTS YAML for mac - Add a function to upload log files during a test on VSTS --- .vsts-ci/mac.yml | 133 ++++++++++++++++++ test/powershell/Host/Logging.Tests.ps1 | 65 +++++---- .../Start-Sleep.Tests.ps1 | 18 +-- .../Modules/HelpersCommon/HelpersCommon.psd1 | 13 +- .../Modules/HelpersCommon/HelpersCommon.psm1 | 44 ++++++ test/tools/Modules/PSSysLog/PSSysLog.psm1 | 64 ++++++++- tools/travis.ps1 | 4 +- 7 files changed, 304 insertions(+), 37 deletions(-) create mode 100644 .vsts-ci/mac.yml diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml new file mode 100644 index 0000000000..019b4e027d --- /dev/null +++ b/.vsts-ci/mac.yml @@ -0,0 +1,133 @@ +name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr) +variables: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + # Avoid expensive initialization of dotnet cli, see: http://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + # Enable VSTS debug mode until stabilitized + system.debug: 'true' + +resources: +- repo: self + clean: true +phases: +- phase: macOS_CI + + queue: + name: Hosted macOS Preview + steps: + - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))" + displayName: Set Build Name for Non-PR + condition: ne(variables['Build.Reason'], 'PullRequest') + +# Several scripts require TMPDIR to exist +# Set it to AGENT_TEMPDIRECTORY so that we know it exists + - powershell: | + Write-Host "##vso[task.setvariable variable=TMPDIR]$env:AGENT_TEMPDIRECTORY" + displayName: Set TMPDIR to AGENT_TEMPDIRECTORY + condition: succeededOrFailed() + + - powershell: | + git submodule update --init + displayName: SubModule Init + condition: succeededOrFailed() + + - powershell: | + rvm install ruby-2.3.3; + rvm --default use 2.3.3; + displayName: Install Ruby + condition: succeededOrFailed() + enabled: false + + - powershell: | + tools/travis.ps1 -Stage Bootstrap + displayName: Bootstrap + condition: succeeded() + + - powershell: | + $ErrorActionPreference = 'continue' + tools/travis.ps1 + displayName: Build and test + condition: succeeded() + + - powershell: | + tools/travis.ps1 -Stage Failure + displayName: After Failure + condition: failed() + + - powershell: | + tools/travis.ps1 -Stage Success + displayName: After Success + condition: succeeded() + + # Uploads any packages as an artifact + - powershell: | + Get-ChildItem -Path *.pkg, *.tar.gz -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { + Write-Host "##vso[artifact.upload containerfolder=artifacts;artifactname=artifacts]$_" + } + displayName: Publish Artifacts + condition: succeededOrFailed() + + # Uploads any Test results as an artifact + - powershell: | + Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { + Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_" + } + displayName: Publish Test Results + condition: succeededOrFailed() + + # Publish the various Test results as Test results to VSTS + # Separate tasks are used because a failure will stop the task + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsSudo.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsSudo.xml' + testRunTitle: Sudo + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsNoSudo.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsNoSudo.xml' + testRunTitle: NoSudo + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsSudo.ExpTest.*.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsSudo.ExpTest.*.xml' + testRunTitle: Sudo Experimental Features + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsNoSudo.ExpTest.*.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsNoSudo.ExpTest.*.xml' + testRunTitle: Sudo Experimental Features + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results ParallelXUnitTestResults.xml + inputs: + testRunner: XUnit + testResultsFiles: '**\ParallelXUnitTestResults.xml' + testRunTitle: Parallel XUnit + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results SequentialXUnitTestResults.xml + inputs: + testRunner: XUnit + testResultsFiles: '**\SequentialXUnitTestResults.xml' + testRunTitle: Sequential XUnit + mergeTestResults: true diff --git a/test/powershell/Host/Logging.Tests.ps1 b/test/powershell/Host/Logging.Tests.ps1 index a53e21f664..fcad56fc59 100644 --- a/test/powershell/Host/Logging.Tests.ps1 +++ b/test/powershell/Host/Logging.Tests.ps1 @@ -5,6 +5,7 @@ using namespace System.Text Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +Import-Module HelpersCommon Import-Module PSSysLog <# @@ -202,35 +203,51 @@ Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix') { } It 'Verifies basic logging with no customizations' -Skip:(!$IsSupportedEnvironment) { - $configFile = WriteLogSettings -LogId $logId - $testPid = & $powershell -NoProfile -SettingsFile $configFile -Command '$PID' + try { + $configFile = WriteLogSettings -LogId $logId + $testPid = & $powershell -NoProfile -SettingsFile $configFile -Command '$PID' - # Made tests more reliable - Start-Sleep -Milliseconds 500 + Export-PSOsLog -After $after -LogPid $testPid -TimeoutInMilliseconds 30000 -IntervalInMilliseconds 3000 -MinimumCount 2 | + Set-Content -Path $contentFile + $items = @(Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 -Verbose) - Export-PSOsLog -After $after -LogPid $testPid -Verbose | Set-Content -Path $contentFile - $items = @(Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 -Verbose) - - $items | Should -Not -Be $null - $items.Count | Should -BeGreaterThan 1 - $items[0].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStart:PowershellConsoleStartup.WinStart.Informational' - $items[1].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStop:PowershellConsoleStartup.WinStop.Informational' - # if there are more items than expected... - if ($items.Count -gt 2) - { - # Force reporting of the first unexpected item to help diagnosis - $items[2] | Should -Be $null + $items | Should -Not -Be $null + $items.Count | Should -BeGreaterThan 1 + $items[0].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStart:PowershellConsoleStartup.WinStart.Informational' + $items[1].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStop:PowershellConsoleStartup.WinStop.Informational' + # if there are more items than expected... + if ($items.Count -gt 2) + { + # Force reporting of the first unexpected item to help diagnosis + $items[2] | Should -Be $null + } + } + catch { + if (Test-Path $contentFile) { + Send-VstsLogFile -Path $contentFile + } + throw } } - It 'Verifies logging level filtering works' -Skip:(!$IsSupportedEnvironment) { - $configFile = WriteLogSettings -LogId $logId -LogLevel Warning - $testPid = & $powershell -NoLogo -NoProfile -SettingsFile $configFile -Command '$PID' + # This is pending because it results in false postitives (-Skip:(!$IsSupportedEnvironment) ) + It 'Verifies logging level filtering works' -Pending { + try { + $configFile = WriteLogSettings -LogId $logId -LogLevel Warning + $testPid = & $powershell -NoLogo -NoProfile -SettingsFile $configFile -Command '$PID' - Export-PSOsLog -After $after -LogPid $testPid -Verbose | Set-Content -Path $contentFile - # by default, powershell startup should only logs informational events. - # With Level = Warning, nothing should be logged. - $items = Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 - $items | Should -Be $null + Export-PSOsLog -After $after -LogPid $testPid | + Set-Content -Path $contentFile + # by default, powershell startup should only logs informational events. + # With Level = Warning, nothing should be logged. + $items = Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 + $items | Should -Be $null + } + catch { + if (Test-Path $contentFile) { + Send-VstsLogFile -Path $contentFile + } + throw + } } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 index f427e7f923..b0d398340f 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 @@ -3,31 +3,33 @@ Describe "Start-Sleep DRT Unit Tests" -Tags "CI" { # WaitHandle.WaitOne(milliseconds, exitContext) is not accurate. - # The wait time varies from 980ms to 1020ms from observation, so + # The wait time varies from 980ms to 1150ms from observation, so # the tests here are changed to be greater than 950ms. - + $minTime = 950 + $maxTime = 1200 + It "Should work properly when sleeping with Second" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -Seconds 1 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } It "Should work properly when sleeping with Milliseconds" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -Milliseconds 1000 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } It "Should work properly when sleeping with ms alias" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -ms 1000 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } } diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 index f811668d6a..f54fa638ba 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 @@ -16,5 +16,16 @@ Copyright = 'Copyright (c) Microsoft Corporation. All rights reserved.' Description = 'Temporary module contains functions for using in tests' -FunctionsToExport = 'Wait-UntilTrue', 'Test-IsElevated', 'Wait-FileToBePresent', 'Get-RandomFileName', 'Enable-Testhook', 'Disable-Testhook', 'Set-TesthookResult', 'Test-TesthookIsSet', 'Add-TestDynamicType' +FunctionsToExport = @( + 'Add-TestDynamicType' + 'Disable-Testhook' + 'Enable-Testhook' + 'Get-RandomFileName' + 'Send-VstsLogFile' + 'Set-TesthookResult' + 'Test-IsElevated' + 'Test-TesthookIsSet' + 'Wait-FileToBePresent' + 'Wait-UntilTrue' + ) } diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index 659f28e8e0..f104e0a2ee 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -192,3 +192,47 @@ public class TestDynamic : DynamicObject } '@ } + +# Upload an artifact in VSTS +# On other systems will just log where the file was placed +function Send-VstsLogFile { + param ( + [parameter(Mandatory,ParameterSetName='contents')] + [string[]] + $Contents, + [parameter(Mandatory,ParameterSetName='contents')] + [string] + $LogName, + [parameter(Mandatory,ParameterSetName='path')] + [ValidateScript({Test-Path -Path $_})] + [string] + $Path + ) + + $logFolder = Join-Path -path $pwd -ChildPath 'logfile' + if(!(Test-Path -Path $logFolder)) + { + $null = New-Item -Path $logFolder -ItemType Directory + if($IsMacOS -or $IsLinux) + { + $null = chmod a+rw $logFolder + } + } + + if($Contents) + { + $logFile = Join-Path -Path $logFolder -ChildPath ([System.Io.Path]::GetRandomFileName() + "-$LogName.txt") + $name = Split-Path -leaf -Path $logFile + + $Contents | out-file -path $logFile -Encoding ascii + } + else + { + $name = Split-Path -leaf -Path $path + $logFile = Join-Path -Path $logFolder -ChildPath ([System.Io.Path]::GetRandomFileName() + '-' + $name) + Copy-Item -Path $Path -Destination $logFile + } + + Write-Host "##vso[artifact.upload containerfolder=$name;artifactname=$name]$logFile" + Write-Verbose "Log file captured as $name" -Verbose +} diff --git a/test/tools/Modules/PSSysLog/PSSysLog.psm1 b/test/tools/Modules/PSSysLog/PSSysLog.psm1 index dcddcf7c02..27f4b5e4ce 100644 --- a/test/tools/Modules/PSSysLog/PSSysLog.psm1 +++ b/test/tools/Modules/PSSysLog/PSSysLog.psm1 @@ -822,7 +822,7 @@ function Get-PSOsLog #> function Export-PSOsLog { - [CmdletBinding()] + [CmdletBinding(DefaultParameterSetName='default')] param ( [Parameter(Mandatory)] @@ -831,7 +831,16 @@ function Export-PSOsLog [string] $LogId = "powershell", - [int] $LogPid + [int] $LogPid, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [int] $TimeoutInMilliseconds, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [int] $IntervalInMilliseconds, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [string] $MinimumCount ) Test-MacOS @@ -863,7 +872,56 @@ function Export-PSOsLog ) } - Start-NativeExecution -command {log show --info @extraParams} + Wait-UntilSuccess { + # Leaving this in an turned on by default until the tests are stabilized. + Write-Verbose "Exporting macOS logs..." -Verbose + $log = @(Start-NativeExecution -command {log show --info @extraParams} | Select-String -SimpleMatch 'com.microsoft.powershell') + + if($log.Count -ge $MinimumCount){ + Write-Output $log + } + else { + throw "did not recieve at least $MinimumCount records but $($log.Count) instead." + } + } -TimeoutInMilliseconds $TimeoutInMilliseconds -IntervalInMilliseconds $IntervalInMilliseconds -LogErrorSb { + $log = Start-NativeExecution -command {log show --info @extraParams} | Select-String -SimpleMatch 'com.microsoft.powershell' + Send-VstsLogFile -Contents $log -LogName 'Export-PSOsLog-Failure' + } +} + +function Wait-UntilSuccess +{ + [CmdletBinding()] + param ( + [ScriptBlock]$sb, + [ScriptBlock]$LogErrorSb, + [int]$TimeoutInMilliseconds = 10000, + [int]$IntervalInMilliseconds = 10000 + ) + # Get the current time + $startTime = [DateTime]::Now + + # Loop until the script block returns + while ($true) { + try{ + return & $sb + } + catch{ + # If the timeout period has passed, return false + $msPassed = ([DateTime]::Now - $startTime).TotalMilliseconds + if ($msPassed -gt $timeoutInMilliseconds) { + if($LogErrorSb) + { + try { & $LogErrorSb } catch {Write-Verbose "Logging of Error details failed with: $_" -Verbose} + } + throw + } + } + + # Sleep for the specified interval + Start-Sleep -Milliseconds $intervalInMilliseconds + } + return $true } <# diff --git a/tools/travis.ps1 b/tools/travis.ps1 index ef43a8c8a2..e0dcb95c43 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -204,12 +204,14 @@ elseif($Stage -eq 'Build') $testResultsNoSudo = "$pwd/TestResultsNoSudo.xml" $testResultsSudo = "$pwd/TestResultsSudo.xml" + $excludeTag = @('RequireSudoOnUnix') + $noSudoPesterParam = @{ 'BinDir' = $output 'PassThru' = $true 'Terse' = $true 'Tag' = @() - 'ExcludeTag' = @('RequireSudoOnUnix') + 'ExcludeTag' = $excludeTag 'OutputFile' = $testResultsNoSudo }