From 1e5655b48cdad707ee3678435c4b587c0856a35c Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 14 Feb 2020 08:44:43 -0800 Subject: [PATCH] Refactor packaging pipeline (#11852) Co-authored-by: Travis Plunk Co-authored-by: Dongbo Wang --- .../releaseBuild/azureDevOps/releaseBuild.yml | 53 +++-- .../templates/windows-hosted-build.yml | 62 ++++++ .../templates/windows-packaging.yml | 199 ++++++++++++++++++ 3 files changed, 301 insertions(+), 13 deletions(-) create mode 100644 tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml create mode 100644 tools/releaseBuild/azureDevOps/templates/windows-packaging.yml diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index c18b19bd10..6fece8bbeb 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -39,42 +39,69 @@ jobs: - template: templates/mac.yml -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: x64 -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: x86 -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: arm -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: arm64 -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: fxdependent -- template: templates/windows-build.yml +- template: templates/windows-hosted-build.yml parameters: Architecture: fxdependentWinDesktop +- template: templates/windows-packaging.yml + parameters: + Architecture: x64 + parentJob: build_windows_x64 -- template: templates/windows-component-governance.yml +- template: templates/windows-packaging.yml + parameters: + Architecture: x86 + parentJob: build_windows_x86 + +- template: templates/windows-packaging.yml + parameters: + Architecture: arm + parentJob: build_windows_arm + +- template: templates/windows-packaging.yml + parameters: + Architecture: arm64 + parentJob: build_windows_arm64 + +- template: templates/windows-packaging.yml + parameters: + Architecture: fxdependent + parentJob: build_windows_fxdependent + +- template: templates/windows-packaging.yml + parameters: + Architecture: fxdependentWinDesktop + parentJob: build_windows_fxdependentWinDesktop - template: templates/windows-package-signing.yml parameters: parentJobs: - - build_windows_x64 - - build_windows_x86 - - build_windows_arm - - build_windows_arm64 - - build_windows_fxdependent - - build_windows_fxdependentWinDesktop + - sign_windows_x64 + - sign_windows_x86 + - sign_windows_arm + - sign_windows_arm64 + - sign_windows_fxdependent + - sign_windows_fxdependentWinDesktop - template: templates/mac-package-signing.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml new file mode 100644 index 0000000000..594513ea22 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -0,0 +1,62 @@ +parameters: + BuildConfiguration: release + BuildPlatform: any cpu + Architecture: x64 + +jobs: +- job: build_windows_${{ parameters.Architecture }} + displayName: Build Windows - ${{ parameters.Architecture }} + condition: succeeded() + pool: + vmImage: windows-latest + variables: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildPlatform: ${{ parameters.BuildPlatform }} + Architecture: ${{ parameters.Architecture }} + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + + steps: + + - checkout: self + clean: true + persistCredentials: true + + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: insert-nuget-config-azfeed.yml + + - powershell: | + Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" + $null = New-Item -ItemType Directory -Path /Powershell -Force + git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell + displayName: Clone PowerShell Repo to /PowerShell + + - powershell: | + + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm" { "win-arm"} + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -location '$(PowerShellRoot)' -destination '$(Build.ArtifactStagingDirectory)/Symbols_$(Architecture)' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' -Symbols + displayName: 'Build Windows Universal - $(Architecture) Symbols zip' + + - powershell: | + $packageName = (Get-ChildItem '$(Build.ArtifactStagingDirectory)\Symbols_$(Architecture)').FullName + $vstsCommandString = "vso[artifact.upload containerfolder=results;artifactname=results]$packageName" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Upload symbols package + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml new file mode 100644 index 0000000000..ccc2ed1ae3 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -0,0 +1,199 @@ +parameters: + BuildConfiguration: release + BuildPlatform: any cpu + Architecture: x64 + parentJob: '' + +jobs: +- job: sign_windows_${{ parameters.Architecture }} + displayName: Package Windows - ${{ parameters.Architecture }} + condition: succeeded() + dependsOn: ${{ parameters.parentJob }} + pool: + name: Package ES Standard Build + variables: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildPlatform: ${{ parameters.BuildPlatform }} + Architecture: ${{ parameters.Architecture }} + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + + steps: + + - checkout: self + clean: true + persistCredentials: true + + - template: shouldSign.yml + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - task: PkgESSetupBuild@10 + displayName: 'Initialize build' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + useDfs: false + productName: PowerShellCore + branchVersion: true + disableWorkspace: true + disableBuildTools: true + disableNugetPack: true + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - powershell: | + $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { + "arm32" + } + else { + '$(Architecture)' + } + + $vstsCommandString = "vso[task.setvariable variable=PkgFilter]$pkgFilter" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set packageName variable + + - task: DownloadBuildArtifacts@0 + inputs: + artifactName: 'results' + itemPattern: '**/*$(PkgFilter).zip' + downloadPath: '$(System.ArtifactsDirectory)\Symbols' + + - powershell: | + Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" + + if ((Test-Path "\PowerShell")) { + Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + + git clone --quiet $env:BUILD_REPOSITORY_LOCALPATH '\PowerShell' + + displayName: Clone PowerShell Repo to /PowerShell + errorActionPreference: silentlycontinue + + - powershell: | + # cleanup previous install + if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { + Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force + } + + $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' + $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix311rtm/wix311-binaries.zip' + Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' + + Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' + Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' + + $msixUrl = '$(makeappUrl)' + Invoke-RestMethod -Uri $msixUrl -OutFile '\makeappx.zip' + Expand-Archive '\makeappx.zip' -destination '\' -Force + + displayName: Install packaging tools + + - powershell: | + $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\results\*$(PkgFilter).zip' + Write-Verbose -Verbose "Zip Path: $zipPath" + + $expandedFolder = $zipPath.BaseName + Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + + Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force + displayName: Expand symbols zip + + - powershell: | + if ("$env:Architecture" -like 'fxdependent*') + { + $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe + } + else + { + $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 + } + displayName: 'Update Signing Xml' + + - task: PkgESCodeSign@10 + displayName: 'CodeSign $(Architecture)' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' + inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' + outPathRoot: '$(System.ArtifactsDirectory)\signed' + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - powershell: | + New-Item -ItemType Directory -Path $(System.ArtifactsDirectory)\signedZip -Force + displayName: 'Create empty signed folder' + condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) + + - powershell: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + + $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' + $destFolder = '$(System.ArtifactsDirectory)\signedZip' + $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' + + New-Item -ItemType Directory -Path $destFolder -Force + + $BuildPackagePath = New-PSSignedBuildZip -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath -DestinationFolder $destFolder + + Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath" + Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath" + + $vstsCommandString = "vso[task.setvariable variable=BuildPackagePath]$BuildPackagePath" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Compress signed files + + - powershell: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm" { "win-arm"} + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + $signedPkg = "$(BuildPackagePath)" + + Write-Verbose -Verbose -Message "signedPkg = $signedPkg" + + $(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -BuildZip $signedPkg -location '$(PowerShellRoot)' -destination '$(System.ArtifactsDirectory)\pkgSigned' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' + displayName: 'Build Windows Universal - $(Architecture) Package' + + - powershell: | + Get-ChildItem '$(System.ArtifactsDirectory)\pkgSigned' | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$packagePath" + } + displayName: Upload packages + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true + + - powershell: | + if ((Test-Path "\PowerShell")) { + Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + + if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { + Write-Verbose -Verbose "Cleaning up Wix tools" + Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force + } + displayName: Clean up local Clone + condition: always()