diff --git a/.spelling b/.spelling index 00783a32cf..2f625c0d1d 100644 --- a/.spelling +++ b/.spelling @@ -75,6 +75,7 @@ nunit exe init labeled +toolset alpha.14 - demos/Azure/README.md alpha.12 @@ -727,3 +728,15 @@ shebang - CHANGELOG.md v6.0.0 alpha.13 + - test/tools/CodeCoverageAutomation/README.md +Start-CodeCoverageRun +Coveralls.io. +v5.0 +v6.0. +CodeCoverage.zip +OpenCover.zip +tests.zip +Coveralls.net +powershell.version +csmacnz +Coveralls.exe diff --git a/test/tools/CodeCoverageAutomation/README.md b/test/tools/CodeCoverageAutomation/README.md new file mode 100644 index 0000000000..928090d646 --- /dev/null +++ b/test/tools/CodeCoverageAutomation/README.md @@ -0,0 +1,23 @@ +# Code Coverage Automation + +The script Start-CodeCoverageRun.ps1 automates the execution of tests and uploading the results to Coveralls.io. + +The script is self contained and can be executed on any Windows system which supports Powershell v5.0 or above. It has not been tested on Powershell v6.0. + +## Execution + +The virtual machine hosting the script has a scheduled task to start the tests at 5 am PDT. + +The script follows the steps below: + +1. Download the code coverage binaries package from AppVeyor nightly builds artifacts (CodeCoverage.zip). +2. Download the OpenCover powershell module from AppVeyor nightly builds artifacts. (OpenCover.zip) +3. Download the tests from AppVeyor nightly builds artifacts (tests.zip) +4. Download Coveralls.net from 'https://github.com/csMACnz/coveralls.net/releases/download/0.7.0/coveralls.net.0.7.0.nupkg' +5. Invoke 'Install-OpenCover' to install OpenCover toolset. +6. Invoke 'Invoke-OpenCover' to execute tests. +7. Invoke powershell to get the git commit ID of the downloaded daily build package. +8. Using the commit ID get committer info like, message, author and email using the github REST API. +9. Invoke 'csmacnz.Coveralls.exe' to upload the coverage results to Coveralls.net + +The uploaded coverage data can be viewed at: https://coveralls.io/github/PowerShell/PowerShell diff --git a/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 b/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 new file mode 100644 index 0000000000..ceeab7929a --- /dev/null +++ b/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 @@ -0,0 +1,125 @@ +param( + [Parameter(Mandatory = $true, Position = 0)] $coverallsToken + ) + +function Write-LogPassThru +{ + Param( + [Parameter(Mandatory=$true, ValueFromPipeline=$true, Position = 0)] + [string] $Message, + $Path = "$env:Temp\CodeCoverageRunLogs.txt" + ) + + $message = "{0:d} - {0:t} : {1}" -f ([datetime]::now),$message + Add-Content -Path $Path -Value $Message -PassThru -Force +} + +Write-LogPassThru -Message "***** New Run *****" + +$codeCoverageZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/CodeCoverage.zip' +$testContentZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/tests.zip' +$openCoverZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/OpenCover.zip' + +Write-LogPassThru -Message "codeCoverageZip: $codeCoverageZip" +Write-LogPassThru -Message "testcontentZip: $testContentZip" +Write-LogPassThru -Message "openCoverZip: $openCoverZip" + +$outputBaseFolder = "$env:Temp\CC" +$null = New-Item -ItemType Directory -Path $outputBaseFolder -Force + +$openCoverPath = "$outputBaseFolder\OpenCover" +$testPath = "$outputBaseFolder\tests" +$psBinPath = "$outputBaseFolder\PSCodeCoverage" +$openCoverTargetDirectory = "$outputBaseFolder\OpenCoverToolset" +$outputLog = "$outputBaseFolder\CodeCoverageOutput.xml" +$psCodeZip = "$outputBaseFolder\PSCode.zip" +$psCodePath = "$outputBaseFolder\PSCode" + +try +{ + $oldErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Stop' + Write-LogPassThru -Message "Starting downloads." + Invoke-WebRequest -uri $codeCoverageZip -outfile "$outputBaseFolder\PSCodeCoverage.zip" + Invoke-WebRequest -uri $testContentZip -outfile "$outputBaseFolder\tests.zip" + Invoke-WebRequest -uri $openCoverZip -outfile "$outputBaseFolder\OpenCover.zip" + Write-LogPassThru -Message "Downloads complete. Starting expansion" + + Expand-Archive -path "$outputBaseFolder\PSCodeCoverage.zip" -destinationpath "$psBinPath" -Force + Expand-Archive -path "$outputBaseFolder\tests.zip" -destinationpath $testPath -Force + Expand-Archive -path "$outputBaseFolder\OpenCover.zip" -destinationpath $openCoverPath -Force + + ## Download Coveralls.net uploader + $coverallsToolsUrl = 'https://github.com/csMACnz/coveralls.net/releases/download/0.7.0/coveralls.net.0.7.0.nupkg' + $coverallsPath = "$outputBaseFolder\coveralls" + + ## Saving the nupkg as zip so we can expand it. + Invoke-WebRequest -uri $coverallsToolsUrl -outfile "$outputBaseFolder\coveralls.zip" + Expand-Archive -Path "$outputBaseFolder\coveralls.zip" -DestinationPath $coverallsPath -Force + + Write-LogPassThru -Message "Expansion complete." + + Import-Module "$openCoverPath\OpenCover" -force + Install-OpenCover -TargetDirectory $openCoverTargetDirectory -force + Write-LogPassThru -Message "OpenCover installed." + + Write-LogPassThru -Message "TestDirectory : $testPath" + Write-LogPassThru -Message "openCoverPath : $openCoverTargetDirectory\OpenCover" + Write-LogPassThru -Message "psbinpath : $psBinPath" + + $openCoverParams = @{outputlog = $outputLog; + TestDirectory = $testPath; + OpenCoverPath = "$openCoverTargetDirectory\OpenCover"; + PowerShellExeDirectory = "$psBinPath\publish" + } + + $openCoverParams | Out-String | Write-LogPassThru + Write-LogPassThru -Message "Starting test run." + + Invoke-OpenCover @openCoverParams + + if(Test-Path $outputLog) + { + Write-LogPassThru -Message (get-childitem $outputLog).FullName + } + + Write-LogPassThru -Message "Test run done." + + $gitCommitId = & "$psBinPath\publish\powershell.exe" -noprofile -command { $PSVersiontable.GitCommitId } + $commitId = $gitCommitId.substring($gitCommitId.LastIndexOf('-g') + 2) + + Write-LogPassThru -Message $commitId + + $coverallsPath = "$outputBaseFolder\coveralls" + + $commitInfo = Invoke-RestMethod -Method Get "https://api.github.com/repos/powershell/powershell/git/commits/$commitId" + $message = ($commitInfo.message).replace("`n", " ") + $author = $commitInfo.author.name + $email = $commitInfo.author.email + + $coverallsExe = Join-Path $coverallsPath "tools\csmacnz.Coveralls.exe" + $coverallsParams = @("--opencover", + "-i $outputLog", + "--repoToken $coverallsToken", + "--commitId $commitId", + "--commitBranch master", + "--commitAuthor `"$author`"", + "--commitEmail $email", + "--commitMessage `"$message`"" + ) + + $coverallsParams | % { Write-LogPassThru -Message $_ } + + & $coverallsExe """$coverallsParams""" + + Write-LogPassThru -Message "Upload complete." +} +catch +{ + $_ +} +finally +{ + Remove-Item -recurse -force -path $outputBaseFolder + $ErrorActionPreference = $oldErrorActionPreference +} \ No newline at end of file