From 1264eedb42e2317113ba222290d5c57e8c624166 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 5 Jan 2017 11:13:22 -0800 Subject: [PATCH] Code coverage automation script (#2918) This script downloads the latest nightly build package and executes the tests on it. It also downloads all necessary tools and then uploads the results to Coveralls.io. The badge for coverage is posted on README.md --- .spelling | 13 ++ test/tools/CodeCoverageAutomation/README.md | 23 ++++ .../Start-CodeCoverageRun.ps1 | 125 ++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 test/tools/CodeCoverageAutomation/README.md create mode 100644 test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 diff --git a/.spelling b/.spelling index 00783a32cf..2f625c0d1d 100644 --- a/.spelling +++ b/.spelling @@ -75,6 +75,7 @@ nunit exe init labeled +toolset alpha.14 - demos/Azure/README.md alpha.12 @@ -727,3 +728,15 @@ shebang - CHANGELOG.md v6.0.0 alpha.13 + - test/tools/CodeCoverageAutomation/README.md +Start-CodeCoverageRun +Coveralls.io. +v5.0 +v6.0. +CodeCoverage.zip +OpenCover.zip +tests.zip +Coveralls.net +powershell.version +csmacnz +Coveralls.exe diff --git a/test/tools/CodeCoverageAutomation/README.md b/test/tools/CodeCoverageAutomation/README.md new file mode 100644 index 0000000000..928090d646 --- /dev/null +++ b/test/tools/CodeCoverageAutomation/README.md @@ -0,0 +1,23 @@ +# Code Coverage Automation + +The script Start-CodeCoverageRun.ps1 automates the execution of tests and uploading the results to Coveralls.io. + +The script is self contained and can be executed on any Windows system which supports Powershell v5.0 or above. It has not been tested on Powershell v6.0. + +## Execution + +The virtual machine hosting the script has a scheduled task to start the tests at 5 am PDT. + +The script follows the steps below: + +1. Download the code coverage binaries package from AppVeyor nightly builds artifacts (CodeCoverage.zip). +2. Download the OpenCover powershell module from AppVeyor nightly builds artifacts. (OpenCover.zip) +3. Download the tests from AppVeyor nightly builds artifacts (tests.zip) +4. Download Coveralls.net from 'https://github.com/csMACnz/coveralls.net/releases/download/0.7.0/coveralls.net.0.7.0.nupkg' +5. Invoke 'Install-OpenCover' to install OpenCover toolset. +6. Invoke 'Invoke-OpenCover' to execute tests. +7. Invoke powershell to get the git commit ID of the downloaded daily build package. +8. Using the commit ID get committer info like, message, author and email using the github REST API. +9. Invoke 'csmacnz.Coveralls.exe' to upload the coverage results to Coveralls.net + +The uploaded coverage data can be viewed at: https://coveralls.io/github/PowerShell/PowerShell diff --git a/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 b/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 new file mode 100644 index 0000000000..ceeab7929a --- /dev/null +++ b/test/tools/CodeCoverageAutomation/Start-CodeCoverageRun.ps1 @@ -0,0 +1,125 @@ +param( + [Parameter(Mandatory = $true, Position = 0)] $coverallsToken + ) + +function Write-LogPassThru +{ + Param( + [Parameter(Mandatory=$true, ValueFromPipeline=$true, Position = 0)] + [string] $Message, + $Path = "$env:Temp\CodeCoverageRunLogs.txt" + ) + + $message = "{0:d} - {0:t} : {1}" -f ([datetime]::now),$message + Add-Content -Path $Path -Value $Message -PassThru -Force +} + +Write-LogPassThru -Message "***** New Run *****" + +$codeCoverageZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/CodeCoverage.zip' +$testContentZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/tests.zip' +$openCoverZip = 'https://ci.appveyor.com/api/projects/PowerShell/powershell-f975h/artifacts/OpenCover.zip' + +Write-LogPassThru -Message "codeCoverageZip: $codeCoverageZip" +Write-LogPassThru -Message "testcontentZip: $testContentZip" +Write-LogPassThru -Message "openCoverZip: $openCoverZip" + +$outputBaseFolder = "$env:Temp\CC" +$null = New-Item -ItemType Directory -Path $outputBaseFolder -Force + +$openCoverPath = "$outputBaseFolder\OpenCover" +$testPath = "$outputBaseFolder\tests" +$psBinPath = "$outputBaseFolder\PSCodeCoverage" +$openCoverTargetDirectory = "$outputBaseFolder\OpenCoverToolset" +$outputLog = "$outputBaseFolder\CodeCoverageOutput.xml" +$psCodeZip = "$outputBaseFolder\PSCode.zip" +$psCodePath = "$outputBaseFolder\PSCode" + +try +{ + $oldErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Stop' + Write-LogPassThru -Message "Starting downloads." + Invoke-WebRequest -uri $codeCoverageZip -outfile "$outputBaseFolder\PSCodeCoverage.zip" + Invoke-WebRequest -uri $testContentZip -outfile "$outputBaseFolder\tests.zip" + Invoke-WebRequest -uri $openCoverZip -outfile "$outputBaseFolder\OpenCover.zip" + Write-LogPassThru -Message "Downloads complete. Starting expansion" + + Expand-Archive -path "$outputBaseFolder\PSCodeCoverage.zip" -destinationpath "$psBinPath" -Force + Expand-Archive -path "$outputBaseFolder\tests.zip" -destinationpath $testPath -Force + Expand-Archive -path "$outputBaseFolder\OpenCover.zip" -destinationpath $openCoverPath -Force + + ## Download Coveralls.net uploader + $coverallsToolsUrl = 'https://github.com/csMACnz/coveralls.net/releases/download/0.7.0/coveralls.net.0.7.0.nupkg' + $coverallsPath = "$outputBaseFolder\coveralls" + + ## Saving the nupkg as zip so we can expand it. + Invoke-WebRequest -uri $coverallsToolsUrl -outfile "$outputBaseFolder\coveralls.zip" + Expand-Archive -Path "$outputBaseFolder\coveralls.zip" -DestinationPath $coverallsPath -Force + + Write-LogPassThru -Message "Expansion complete." + + Import-Module "$openCoverPath\OpenCover" -force + Install-OpenCover -TargetDirectory $openCoverTargetDirectory -force + Write-LogPassThru -Message "OpenCover installed." + + Write-LogPassThru -Message "TestDirectory : $testPath" + Write-LogPassThru -Message "openCoverPath : $openCoverTargetDirectory\OpenCover" + Write-LogPassThru -Message "psbinpath : $psBinPath" + + $openCoverParams = @{outputlog = $outputLog; + TestDirectory = $testPath; + OpenCoverPath = "$openCoverTargetDirectory\OpenCover"; + PowerShellExeDirectory = "$psBinPath\publish" + } + + $openCoverParams | Out-String | Write-LogPassThru + Write-LogPassThru -Message "Starting test run." + + Invoke-OpenCover @openCoverParams + + if(Test-Path $outputLog) + { + Write-LogPassThru -Message (get-childitem $outputLog).FullName + } + + Write-LogPassThru -Message "Test run done." + + $gitCommitId = & "$psBinPath\publish\powershell.exe" -noprofile -command { $PSVersiontable.GitCommitId } + $commitId = $gitCommitId.substring($gitCommitId.LastIndexOf('-g') + 2) + + Write-LogPassThru -Message $commitId + + $coverallsPath = "$outputBaseFolder\coveralls" + + $commitInfo = Invoke-RestMethod -Method Get "https://api.github.com/repos/powershell/powershell/git/commits/$commitId" + $message = ($commitInfo.message).replace("`n", " ") + $author = $commitInfo.author.name + $email = $commitInfo.author.email + + $coverallsExe = Join-Path $coverallsPath "tools\csmacnz.Coveralls.exe" + $coverallsParams = @("--opencover", + "-i $outputLog", + "--repoToken $coverallsToken", + "--commitId $commitId", + "--commitBranch master", + "--commitAuthor `"$author`"", + "--commitEmail $email", + "--commitMessage `"$message`"" + ) + + $coverallsParams | % { Write-LogPassThru -Message $_ } + + & $coverallsExe """$coverallsParams""" + + Write-LogPassThru -Message "Upload complete." +} +catch +{ + $_ +} +finally +{ + Remove-Item -recurse -force -path $outputBaseFolder + $ErrorActionPreference = $oldErrorActionPreference +} \ No newline at end of file