diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 156f747a16..94389f6175 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,6 +8,7 @@ "ms-vscode.PowerShell", "twxs.cmake", "DavidAnson.vscode-markdownlint", - "ms-azure-devops.azure-pipelines" + "ms-azure-devops.azure-pipelines", + "vitaliymaz.vscode-svg-previewer" ] } diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 40ed84463f..fd9aade734 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -77,7 +77,7 @@ jobs: displayName: Set TEMP to AGENT_TEMPDIRECTORY condition: succeededOrFailed() - - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config.yml + - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - powershell: | tools/travis.ps1 -Stage Bootstrap diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 9a77e936b3..f9e8348fa3 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -52,7 +52,7 @@ jobs: displayName: Set Build Name for Non-PR condition: ne(variables['Build.Reason'], 'PullRequest') - - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config.yml + - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - powershell: | tools/travis.ps1 -Stage Bootstrap diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index e36dd6ade9..f27eb15055 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -54,7 +54,7 @@ steps: displayName: Set Build Name for Non-PR condition: ne(variables['Build.Reason'], 'PullRequest') - - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config.yml + - template: ../tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml - powershell: | [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 diff --git a/tools/releaseBuild/README.md b/tools/releaseBuild/README.md index 838933702f..9b78e742b5 100644 --- a/tools/releaseBuild/README.md +++ b/tools/releaseBuild/README.md @@ -1,33 +1,55 @@ -# VSTS Release Builds +# Azure Dev Ops Release Builds ## Requirements Docker must be installed to run any of the release builds. -## Running Windows Release Builds +## Running Windows Release Builds Locally From PowerShell on Windows, run `.\vstsbuild.ps1 -ReleaseTag -Name `. +For the package builds, run `.\vstsbuild.ps1 -ReleaseTag -Name -BuildPath -SignedFilesPath ` + Windows Build Names: - * `win7-x64` - * Builds the Universal Windows x64 Package - * `win7-x86` - * Builds the Universal Windows x86 Package - * `win7-x64-symbols` - * Builds the Windows x64 Zip with symbols - * `win7-x86-symbols` - * Builds the Windows x86 Zip with symbols +* `win7-x64-symbols` + * Builds the Windows x64 Zip with symbols +* `win7-x86-symbols` + * Builds the Windows x86 Zip with symbols +* `win7-arm-symbols` + * Builds the Windows ARM Zip with symbols +* `win7-arm64-symbols` + * Builds the Windows ARM64 Zip with symbols +* `win7-fxdependent-symbols` + * Builds the Windows FxDependent Zip with symbols +* `win7-x64-package` + * Builds the Windows x64 packages +* `win7-x86-package` + * Builds the Windows x86 packages +* `win7-arm-package` + * Builds the Windows ARM packages +* `win7-arm64-package` + * Builds the Windows ARM64 packages +* `win7-fxdependent-package` + * Builds the Windows FxDependent packages -## Running Linux Release Builds +## Running Linux Release Builds Locally From PowerShell on Linux or macOS, run `.\vstsbuild.ps1 -ReleaseTag -Name `. Linux Build Names: - * `ubuntu.14.04` - * Builds the Ubuntu 14.04 Package and AppImage Package - * `ubuntu.16.04` - * Builds the Ubuntu 16.04 Package - * `centos.7` - * Builds the CentOS 7 Package +* `deb` + * Builds the Debian Packages, ARM32 and ARM64. +* `alpine` + * Builds the Alpine Package +* `rpm` + * Builds the RedHat variant Package + +## Azure Dev Ops Build + +The release build is fairly complicated. The definition is at `./azureDevOps/releaseBuild.yml`. + +Here is a diagram of the build: + +[![Release Build diagram](https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/releaseBuild/azureDevOps/diagram.svg?sanitize=true)](https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/releaseBuild/azureDevOps/diagram.svg?sanitize=true) diff --git a/tools/releaseBuild/azureDevOps/diagram.png b/tools/releaseBuild/azureDevOps/diagram.png deleted file mode 100644 index 0c53297198..0000000000 Binary files a/tools/releaseBuild/azureDevOps/diagram.png and /dev/null differ diff --git a/tools/releaseBuild/azureDevOps/diagram.puml b/tools/releaseBuild/azureDevOps/diagram.puml index f2593038f0..ad5da432b8 100644 --- a/tools/releaseBuild/azureDevOps/diagram.puml +++ b/tools/releaseBuild/azureDevOps/diagram.puml @@ -1,31 +1,50 @@ @startuml -' Define the build tasks as business processes -archimate #Business "Build DEB" as BuildDEB <> -archimate #Business "Build RPM" as BuildRPM <> -archimate #Business "Build Alpine" as BuildAlpine <> -archimate #Business "Build Linux-FxDependent" as BuildLinuxFx <> -archimate #Business "Upload DEB" as UploadDEB <> -archimate #Business "Upload RPM" as UploadRPM <> -archimate #Business "Upload Alpine" as UploadAlpine <> -archimate #Business "Upload Linux-FxDependent" as UploadLinuxFx <> +folder "Linux Builds" { + ' Define the build tasks as business processes + agent "DEB" as BuildDEB + agent "RPM" as BuildRPM + agent "Alpine" as BuildAlpine + agent "Linux-FxDependent" as BuildLinuxFx -archimate #Business "Build macOS" as BuildMac <> -archimate #Business "macOS Package Signing and Upload" as SignMac <> +} -archimate #Business "Build Windows - x64" as BuildWinX64 <> -archimate #Business "Build Windows - x86" as BuildWinX86 <> -archimate #Business "Build Windows - arm32" as BuildWinArm32 <> -archimate #Business "Build Windows - arm64" as BuildWinArm64 <> -archimate #Business "Build Windows - FxDependent" as BuildWinFx <> -archimate #Business "Windows Package Signing and Upload" as SignWin <> +agent "macOS Build" as BuildMac -archimate #Business "ComponentRegistration" as BuildCG <> +folder "Windows Builds" as WinBuilds { + agent "x64" as BuildWinX64 + agent "x86" as BuildWinX86 + agent "arm32" as BuildWinArm32 + agent "arm64" as BuildWinArm64 + agent "FxDependent" as BuildWinFx +} -archimate #Business "Compliance" as Compliance <> +agent "ComponentRegistration" as BuildCG + +folder "Linux Package Scanning and Upload" { + agent "DEB" as UploadDEB + agent "RPM" as UploadRPM + agent "Alpine" as UploadAlpine + agent "Linux-FxDependent" as UploadLinuxFx +} + +folder "Package Signing and Upload" { + agent "macOS" as SignMac + + agent "Windows" as SignWin +} + + +agent "Compliance" as Compliance + +folder "Create & Upload FxDependent DotNet SDK" { + agent "Linux" as BuildLinuxFxSdk + agent "Windows" as BuildWinFxSdk +} ' Define finishing the build as a goal filled -archimate #Goal "Finish" as Finish <> +control "Finish" as Finish +control "Start" as Start ' map the various Upload task dependencies BuildDEB -down-> UploadDEB @@ -37,25 +56,42 @@ BuildAlpine -down-> UploadAlpine BuildMac -down-> SignMac ' map all of the SignWin task dependencies -BuildWinX64 -down-> SignWin -BuildWinX86 -down-> SignWin -BuildWinArm32 -down-> SignWin -BuildWinArm64 -down-> SignWin -BuildWinFx -down-> SignWin +WinBuilds -down-> SignWin +'BuildWinX64 -down-> SignWin +'BuildWinX86 -down-> SignWin +'BuildWinArm32 -down-> SignWin +'BuildWinArm64 -down-> SignWin +'BuildWinFx -down-> SignWin ' map all of the Compliance task dependencies BuildWinX86 -down-> Compliance BuildWinX64 -down-> Compliance BuildWinFx -down-> Compliance +BuildLinuxFx -down-> BuildLinuxFxSdk +SignWin -down-> BuildWinFxSdk + ' map all leafs to finish -Compliance -down-> Finish -UploadAlpine -down-> Finish -UploadDEB -down-> Finish -UploadRPM -down-> Finish -UploadLinuxFx -down-> Finish -SignWin -down-> Finish -SignMac -down-> Finish -BuildCG -down-> Finish +Compliance ~~ Finish +UploadAlpine ~~ Finish +UploadDEB ~~ Finish +UploadRPM ~~ Finish +UploadLinuxFx ~~ Finish +SignMac ~~ Finish +BuildCG ~~ Finish +BuildLinuxFxSdk ~~ Finish +BuildWinFxSdk ~~ Finish + +Start ~~ BuildDEB +Start ~~ BuildRPM +Start ~~ BuildAlpine +Start ~~ BuildLinuxFx +Start ~~ BuildMac +Start ~~ BuildWinX64 +Start ~~ BuildWinX86 +Start ~~ BuildWinFx +Start ~~ BuildWinArm32 +Start ~~ BuildWinArm64 +Start ~~ BuildCG @enduml diff --git a/tools/releaseBuild/azureDevOps/diagram.svg b/tools/releaseBuild/azureDevOps/diagram.svg new file mode 100644 index 0000000000..f17b357389 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/diagram.svg @@ -0,0 +1,98 @@ +Linux BuildsWindows BuildsLinux Package Scanning and UploadPackage Signing and UploadCreate & Upload FxDependent DotNet SDKDEBRPMAlpineLinux-FxDependentx64x86arm32arm64FxDependentDEBRPMAlpineLinux-FxDependentmacOSWindowsLinuxWindowsmacOS BuildComponentRegistrationComplianceFinishStart \ No newline at end of file diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 9255c247bd..f0d331423e 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -8,6 +8,8 @@ jobs: ${{ parameters.parentJobs }} pool: name: Package ES CodeHub Lab E + # APIScan can take a long time + timeoutInMinutes: 90 steps: - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml similarity index 100% rename from tools/releaseBuild/azureDevOps/templates/insert-nuget-config.yml rename to tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 3da9d24062..dbe768258b 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -27,7 +27,7 @@ jobs: } displayName: 'Skip Alpine or fxdependent for PS v6.0.*' - - template: insert-nuget-config.yml + - template: insert-nuget-config-azfeed.yml - powershell: | @@ -81,7 +81,7 @@ jobs: displayName: 'Run Defender Scan' - task: AzureFileCopy@1 - displayName: 'Upload to azure deb and tar.gz' + displayName: 'Upload to Azure - DEB and tar.gz' inputs: SourcePath: '$(System.ArtifactsDirectory)\finished\release' azureSubscription: '$(AzureFileCopySubscription)' @@ -89,8 +89,12 @@ jobs: storage: '$(StorageAccount)' ContainerName: '$(AzureVersion)' + - template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory)\finished\release + - task: AzureFileCopy@1 - displayName: 'Upload to azure rpm' + displayName: 'Upload to Azure - RPM' inputs: SourcePath: '$(System.ArtifactsDirectory)\rpm\release' azureSubscription: '$(AzureFileCopySubscription)' @@ -98,3 +102,7 @@ jobs: storage: '$(StorageAccount)' ContainerName: '$(AzureVersion)-unsigned' condition: and(eq(variables['buildName'], 'RPM'),succeeded()) + + - template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory)\rpm\release diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 68c536097e..f9f8fd7c72 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -52,19 +52,10 @@ jobs: signConfigXml: '$(System.ArtifactsDirectory)\package.xml' outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - - powershell: | - dir "$(System.ArtifactsDirectory)\*.tar.gz" -Recurse | foreach-object { - $fullname = $_.fullname - Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$fullname" - } - displayName: 'Upload tar.gz' - - - powershell: | - dir "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | foreach-object { - $fullname = $_.fullname - Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$fullname" - } - displayName: 'Upload macOS signed package' + - template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory) + artifactFilter: "*.tar.gz" - powershell: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" @@ -73,7 +64,12 @@ jobs: Expand-Archive -Path $zipPath -DestinationPath $destination $targzPath = dir "$(System.ArtifactsDirectory)\*.tar.gz" -Recurse | select-object -expandproperty fullname Copy-Item -Path $targzPath -Destination $destination - displayName: 'Extract and copy macOS artifacts for upload to azure' + displayName: 'Extract and copy macOS artifacts for upload' + + - template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory)\azureMacOs + artifactFilter: "*.pkg" - task: AzureFileCopy@1 displayName: 'AzureBlob File Copy' diff --git a/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml b/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml new file mode 100644 index 0000000000..287ca50ba5 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml @@ -0,0 +1,14 @@ +parameters: + artifactPath: + artifactFilter: '*' + condition: succeeded() + +steps: + - powershell: | + Get-ChildItem -Path '${{ parameters.artifactPath }}' -File -filter '${{ parameters.artifactFilter }}' -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty FullName | + ForEach-Object { + Write-Host "##vso[artifact.upload containerfolder=finalResults;artifactname=finalResults]$_" + } + displayName: Upload Final Artifacts ${{ parameters.artifactFilter }} from ${{ parameters.artifactPath }} + condition: ${{ parameters.condition }} diff --git a/tools/releaseBuild/azureDevOps/templates/upload.yml b/tools/releaseBuild/azureDevOps/templates/upload.yml index 8003917d07..f1402eb15c 100644 --- a/tools/releaseBuild/azureDevOps/templates/upload.yml +++ b/tools/releaseBuild/azureDevOps/templates/upload.yml @@ -4,14 +4,11 @@ parameters: msi: yes steps: -- powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed msi - ${{ parameters.architecture }}' - condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) +- template: upload-final-results.yml + parameters: + artifactPath: $(Build.StagingDirectory)\signedPackages + artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msi + condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) - task: AzureFileCopy@1 displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}' @@ -23,13 +20,10 @@ steps: ContainerName: '$(AzureVersion)' condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) -- powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - ${{ parameters.architecture }}' +- template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory)\signed + artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.zip - task: AzureFileCopy@1 displayName: 'upload signed zip to Azure - ${{ parameters.architecture }}' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-build.yml index 2aa279178d..6b457cfd78 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-build.yml @@ -53,7 +53,7 @@ jobs: Write-Host -Object "##$vstsCommandString" displayName: 'Skip FxDependent for PS v6.0.*' - - template: insert-nuget-config.yml + - template: insert-nuget-config-azfeed.yml - powershell: | docker container prune --force diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index c40cd08062..d4698f84a4 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -49,7 +49,7 @@ jobs: # Diagnostics is not critical it passes every time it runs continueOnError: true - - template: insert-nuget-config.yml + - template: insert-nuget-config-azfeed.yml - powershell: | ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-x64-component-registration