From 414d56170b7abe7b602d1692de4b570671c8b7e6 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 26 Sep 2017 14:02:55 -0700 Subject: [PATCH] Move release build definition into PowerShell (#4884) * Add docker related files from PSRelease base release docker builds on PowerShell release docker images * ignore the PSRelease folder * Add vstsbuild.sh wrapper of vstsbuild.ps1 * add vstsbuild.ps1 * Add build.json * address PR feedbock --- tools/releaseBuild/.gitignore | 1 + .../GenericLinuxFiles/PowerShellPackage.ps1 | 57 +++++++++++ .../microsoft_powershell_centos7/Dockerfile | 27 ++++++ .../Dockerfile | 36 +++++++ .../Dockerfile | 34 +++++++ .../DockerFile | 11 +++ .../PowerShellPackage.ps1 | 95 +++++++++++++++++++ tools/releaseBuild/build.json | 63 ++++++++++++ tools/releaseBuild/vstsbuild.ps1 | 59 ++++++++++++ tools/releaseBuild/vstsbuild.sh | 1 + 10 files changed, 384 insertions(+) create mode 100644 tools/releaseBuild/.gitignore create mode 100644 tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 create mode 100644 tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile create mode 100644 tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile create mode 100644 tools/releaseBuild/Images/microsoft_powershell_ubuntu16.04/Dockerfile create mode 100644 tools/releaseBuild/Images/microsoft_powershell_windowsservercore/DockerFile create mode 100644 tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 create mode 100644 tools/releaseBuild/build.json create mode 100644 tools/releaseBuild/vstsbuild.ps1 create mode 100644 tools/releaseBuild/vstsbuild.sh diff --git a/tools/releaseBuild/.gitignore b/tools/releaseBuild/.gitignore new file mode 100644 index 0000000000..0ff566888a --- /dev/null +++ b/tools/releaseBuild/.gitignore @@ -0,0 +1 @@ +PSRelease/ diff --git a/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 b/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 new file mode 100644 index 0000000000..21b19a8c57 --- /dev/null +++ b/tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1 @@ -0,0 +1,57 @@ +# PowerShell Script to build and package PowerShell from specified form and branch +# Script is intented to use in Docker containers +# Ensure PowerShell is available in the provided image + +param ( + [string] $location = "/powershell", + + # Destination location of the package on docker host + [string] $destination = '/mnt', + + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+\.\d+)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag, + [switch]$AppImage +) + +$releaseTagParam = @{} +if($ReleaseTag) +{ + $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } +} + +Push-Location +try { + Set-Location $location + Import-Module "$location/build.psm1" + Import-Module "$location/tools/packaging" + + Start-PSBootstrap -Package -NoSudo + Start-PSBuild -Crossgen -PSModuleRestore @releaseTagParam + + Start-PSPackage @releaseTagParam + if($AppImage.IsPresent) + { + Start-PSPackage -Type AppImage @releaseTagParam + } +} +finally +{ + Pop-Location +} + +$linuxPackages = Get-ChildItem "$location/powershell*" -Include *.deb,*.rpm + +foreach($linuxPackage in $linuxPackages) +{ + Copy-Item -Path $linuxPackage.FullName -Destination $destination -force +} + +if($AppImage.IsPresent) +{ + $appImages = Get-ChildItem -Path $location -Filter '*.AppImage' + foreach($appImageFile in $appImages) + { + Copy-Item -Path $appImageFile.FullName -Destination $destination -force + } +} diff --git a/tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile new file mode 100644 index 0000000000..a49a7cc8f0 --- /dev/null +++ b/tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile @@ -0,0 +1,27 @@ +# Docker image file that describes an Centos7 image with PowerShell installed from Microsoft YUM Repo + +FROM microsoft/powershell:centos7 +LABEL maintainer="PowerShell Team " + +# Install dependencies and clean up +RUN yum install -y \ + glibc \ + libcurl \ + ca-certificates \ + libgcc \ + libicu \ + openssl \ + libstdc++ \ + ncurses-base \ + libunwind \ + uuid \ + zlib \ + which \ + curl \ + git \ + && yum clean all + +COPY PowerShellPackage.ps1 / + +# Use PowerShell as the default shell +ENTRYPOINT [ "powershell" ] diff --git a/tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile new file mode 100644 index 0000000000..7337660537 --- /dev/null +++ b/tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile @@ -0,0 +1,36 @@ +# Docker image file that describes an Ubuntu14.04 image with PowerShell installed from Microsoft APT Repo + +FROM microsoft/powershell:ubuntu14.04 +LABEL maintainer="PowerShell Team " + +# Install dependencies and clean up +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-utils \ + libc6 \ + libcurl3 \ + ca-certificates \ + libgcc1 \ + libicu52 \ + libssl1.0.0 \ + libstdc++6 \ + libtinfo5 \ + libunwind8 \ + libuuid1 \ + libcroco3 \ + libgraphite2-3 \ + zlib1g \ + curl \ + git \ + apt-transport-https \ + wget \ + dpkg-dev \ + libfuse-dev \ + fuse \ + python \ + && rm -rf /var/lib/apt/lists/* + +COPY PowerShellPackage.ps1 / + +# Use PowerShell as the default shell +ENTRYPOINT [ "powershell" ] diff --git a/tools/releaseBuild/Images/microsoft_powershell_ubuntu16.04/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_ubuntu16.04/Dockerfile new file mode 100644 index 0000000000..cfd7c076e9 --- /dev/null +++ b/tools/releaseBuild/Images/microsoft_powershell_ubuntu16.04/Dockerfile @@ -0,0 +1,34 @@ +# Docker image file that describes an Ubuntu16.04 image with PowerShell installed from Microsoft APT Repo + +FROM microsoft/powershell:ubuntu16.04 +LABEL maintainer="PowerShell Team " + +# Install dependencies and clean up +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-utils \ + apt-utils \ + libc6 \ + libcurl3 \ + ca-certificates \ + libgcc1 \ + libicu55 \ + libssl1.0.0 \ + libstdc++6 \ + libtinfo5 \ + libunwind8 \ + libuuid1 \ + libcroco3 \ + libgraphite2-3 \ + zlib1g \ + curl \ + git \ + apt-transport-https \ + locales \ + wget \ + && rm -rf /var/lib/apt/lists/* + +COPY PowerShellPackage.ps1 / + +# Use PowerShell as the default shell +ENTRYPOINT [ "powershell" ] diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/DockerFile b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/DockerFile new file mode 100644 index 0000000000..883d4c34d0 --- /dev/null +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/DockerFile @@ -0,0 +1,11 @@ +# escape=` +#0.3.6 (no powershell 6) +FROM travisez13/microsoft.windowsservercore.build-tools:latest +LABEL maintainer='PowerShell Team ' +LABEL description="This Dockerfile for Windows Server Core with git installed via chocolatey." + +SHELL ["powershell"] + +COPY PowerShellPackage.ps1 / + +ENTRYPOINT ["powershell"] diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 new file mode 100644 index 0000000000..2c5a7fd713 --- /dev/null +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 @@ -0,0 +1,95 @@ +[cmdletbinding()] +# PowerShell Script to clone, build and package PowerShell from specified fork and branch +param ( + [string] $fork = 'powershell', + [string] $branch = 'master', + [string] $location = "$pwd\powershell", + [string] $destination = "$env:WORKSPACE", + [ValidateSet("win7-x64", "win81-x64", "win10-x64", "win7-x86")] + [string]$Runtime = 'win10-x64', + [switch] $Wait, + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+\.\d+)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag +) + +$releaseTagParam = @{} +if($ReleaseTag) +{ + $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } +} + +if(-not $env:homedrive) +{ + Write-Verbose "fixing empty home paths..." -Verbose + $profileParts = $env:userprofile -split ':' + $env:homedrive = $profileParts[0]+':' + $env:homepath = $profileParts[1] +} + +if(! (Test-Path $destination)) +{ + Write-Verbose "Creating destination $destination" -Verbose + $null = New-Item -Path $destination -ItemType Directory +} + +Write-Verbose "homedrive : ${env:homedrive}" +Write-Verbose "homepath : ${env:homepath}" + +# Don't use CIM_PhysicalMemory, docker containers may cache old values +$memoryMB = (Get-CimInstance win32_computersystem).TotalPhysicalMemory /1MB +$requiredMemoryMB = 2048 +if($memoryMB -lt $requiredMemoryMB) +{ + throw "Building powershell requires at least $requiredMemoryMB MiB of memory and only $memoryMB MiB is present." +} +Write-Verbose "Running with $memoryMB MB memory." -Verbose + +try{ + Set-Location $location + + Import-Module "$location\build.psm1" -Force + Import-Module "$location\tools\packaging" -Force + $env:platform = $null + Write-Verbose "Bootstrapping powershell build..." -verbose + Start-PSBootstrap -Force -Package + + Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -verbose + Start-PSBuild -Clean -CrossGen -PSModuleRestore -Runtime $Runtime -Configuration Release @releaseTagParam + + $pspackageParams = @{'Type'='msi'} + if ($Runtime -ne 'win10-x64') + { + $pspackageParams += @{'WindowsRuntime'=$Runtime} + } + + Write-Verbose "Starting powershell packaging(msi)..." -verbose + Start-PSPackage @pspackageParams @releaseTagParam + + $pspackageParams['Type']='zip' + Write-Verbose "Starting powershell packaging(zip)..." -verbose + Start-PSPackage @pspackageParams @releaseTagParam + + Write-Verbose "Exporting packages ..." -verbose + + Get-ChildItem $location\*.msi,$location\*.zip | Select-Object -ExpandProperty FullName | ForEach-Object { + $file = $_ + Write-Verbose "Copying $file to $destination" -verbose + Copy-Item -Path $file -Destination "$destination\" -Force + } +} +finally +{ + Write-Verbose "Beginning build clean-up..." -verbose + if($Wait.IsPresent) + { + $path = Join-Path $PSScriptRoot -ChildPath 'delete-to-continue.txt' + $null = New-Item -Path $path -ItemType File + Write-Verbose "Computer name: $env:COMPUTERNAME" -Verbose + Write-Verbose "Delete $path to exit." -Verbose + while(Test-Path -LiteralPath $path) + { + Start-Sleep -Seconds 60 + } + } +} diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json new file mode 100644 index 0000000000..82565c9a14 --- /dev/null +++ b/tools/releaseBuild/build.json @@ -0,0 +1,63 @@ +{ + "Windows": [ + { + "Name": "win7-x64", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1"], + "DockerImageName": "ps-winsrvcore" + }, + { + "Name": "win7-x86", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1"], + "DockerImageName": "ps-winsrvcore" + } + ], + "Linux": [ + { + "Name": "ubuntu.14.04", + "RepoDestinationPath": "/PowerShell", + "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_ -AppImage", + "BuildDockerOptions": [ + "--cap-add", + "SYS_ADMIN", + "--cap-add", + "MKNOD", + "--device=/dev/fuse", + "--security-opt", + "apparmor:unconfined" + ], + "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile", + "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], + "DockerImageName": "ps-ubunutu-14-04" + }, + { + "Name": "ubuntu.16.04", + "RepoDestinationPath": "/PowerShell", + "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_", + "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], + "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_ubuntu16.04/Dockerfile", + "DockerImageName": "ps-ubunutu-16-04" + }, + { + "Name": "centos.7", + "RepoDestinationPath": "/PowerShell", + "BuildCommand": "/PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -ReleaseTag _ReleaseTag_", + "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"], + "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile", + "DockerImageName": "ps-centos-7" + } + ] +} diff --git a/tools/releaseBuild/vstsbuild.ps1 b/tools/releaseBuild/vstsbuild.ps1 new file mode 100644 index 0000000000..9ef0d4e90c --- /dev/null +++ b/tools/releaseBuild/vstsbuild.ps1 @@ -0,0 +1,59 @@ +param( + [Parameter(ParameterSetName='Build')] + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+\.\d+)?$")] + [string]$ReleaseTag, + + [ValidateSet('win7-x86','win7-x64','ubuntu.14.04','ubuntu.16.04','centos.7')] + [String] + $Name +) +$ErrorActionPreference = 'Stop' + + +$psReleaseBranch = 'master' +$psReleaseFork = 'PowerShell' +$location = Join-Path -Path $PSScriptRoot -ChildPath 'PSRelease' +if(Test-Path $location) +{ + Remove-Item -Path $location -Recurse -Force +} + +$gitBinFullPath = (Get-Command -Name git).Source +if (-not $gitBinFullPath) +{ + throw "Git is required to proceed. Install from 'https://git-scm.com/download/win'" +} + +Write-Verbose "cloning -b $psReleaseBranch --quiet https://github.com/$psReleaseFork/PSRelease.git" -verbose +& $gitBinFullPath clone -b $psReleaseBranch --quiet https://github.com/$psReleaseFork/PSRelease.git $location + +Push-Location -Path $PWD.Path +try{ + Set-Location $location + & $gitBinFullPath submodule update --init --recursive --quiet +} +finally +{ + Pop-Location +} + +$unresolvedRepoRoot = Join-Path -Path $PSScriptRoot '../..' +$resolvedRepoRoot = (Resolve-Path -Path $unresolvedRepoRoot).ProviderPath + +try +{ + Write-Verbose "Starting build at $resolvedRepoRoot ..." -Verbose + Import-Module "$location/vstsBuild" -Force + Import-Module "$location/dockerBasedBuild" -Force + Clear-VstsTaskState + + Invoke-Build -RepoPath $resolvedRepoRoot -BuildJsonPath './tools/releaseBuild/build.json' -Name $Name -Parameters $PSBoundParameters +} +catch +{ + Write-VstsError -Error $_ +} +finally{ + Write-VstsTaskState + exit 0 +} diff --git a/tools/releaseBuild/vstsbuild.sh b/tools/releaseBuild/vstsbuild.sh new file mode 100644 index 0000000000..d805992aa4 --- /dev/null +++ b/tools/releaseBuild/vstsbuild.sh @@ -0,0 +1 @@ +powershell -command ".\vstsbuild.ps1 $*"