Performance improvements for release build (#9179)

This PR optimizes the download tasks to reduce build times. After the change we see a order of magnitude reduction in download times for Windows package signing and also the compliance tasks.

In addition to that the PR also makes changes to organize the build artifacts in folders.

tools/releaseBuild/azureDevOps/diagram.puml

@@ -11,6 +11,8 @@ folder "Linux Builds" as LinuxBuilds {
 
 agent "macOS Build"  as BuildMac
 
+agent "Upload build metadata" as BuildMetadata
+
 folder "Windows Builds" as WinBuilds {
     agent "x64"  as BuildWinX64
     agent "x86"  as BuildWinX86
@@ -84,6 +86,7 @@ SignMac ~~ Finish
 BuildCG ~~ Finish
 BuildNuGet ~~ Finish
 TestArtifacts ~~ Finish
+BuildMetadata ~~ Finish
 
 Start ~~ BuildDEB
 Start ~~ BuildRPM
@@ -97,5 +100,6 @@ Start ~~ BuildWinArm32
 Start ~~ BuildWinArm64
 Start ~~ BuildCG
 Start ~~ TestArtifacts
+Start ~~ BuildMetadata
 
 @enduml

tools/releaseBuild/azureDevOps/releaseBuild.yml

@@ -96,3 +96,16 @@ jobs:
   parameters:
     buildName: linux
     pool: Hosted Ubuntu 1604
+
+- job: release_json
+  displayName: Create and Upload release.json
+  steps:
+  - template: templates/SetVersionVariables.yml
+    parameters:
+      ReleaseTagVar: $(ReleaseTagVar)
+
+  - powershell: |
+      @{ ReleaseVersion = "$(Version)" } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json"
+      Get-Content "$(Build.StagingDirectory)\release.json"
+      Write-Host "##vso[artifact.upload containerfolder=metadata;artifactname=metadata]$(Build.StagingDirectory)\release.json"
+    displayName: Create and upload release.json file to build artifact

tools/releaseBuild/azureDevOps/templates/compliance.yml

@@ -20,9 +20,10 @@ jobs:
   - task: DownloadBuildArtifacts@0
     displayName: 'Download artifacts'
     inputs:
-      downloadType: specific
-      itemPattern: |
-        **/*.zip
+      buildType: current
+      downloadType: single
+      artifactName: results
+      downloadPath: '$(System.ArtifactsDirectory)'
 
   - powershell: |
       dir "$(System.ArtifactsDirectory)\*" -Recurse

tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml

@@ -3,7 +3,7 @@ parameters:
 
 jobs:
 - job: WinPackageSigningJob
-  displayName: Windows Package signing
+  displayName: Windows Package signing and upload
   dependsOn:
     ${{ parameters.parentJobs }}
   condition: succeeded()
@@ -22,10 +22,10 @@ jobs:
   - task: DownloadBuildArtifacts@0
     displayName: 'Download artifacts'
     inputs:
-      downloadType: specific
-      itemPattern: |
-        **/*.zip
-        **/*.msi
+      buildType: current
+      downloadType: single
+      artifactName: signed
+      downloadPath: '$(System.ArtifactsDirectory)'
 
   - powershell: |
       dir "$(System.ArtifactsDirectory)\*" -Recurse
@@ -33,18 +33,6 @@ jobs:
     # Diagnostics is not critical it passes every time it runs
     continueOnError: true
 
-  - powershell: |
-      Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x86.zip" -Destination "$(Build.StagingDirectory)\symbols\x86"
-    displayName: 'Expand symbols zip - x86'
-
-  - powershell: |
-      Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x64.zip" -Destination "$(Build.StagingDirectory)\symbols\x64"
-    displayName: 'Expand symbols zip - x64'
-
-  - powershell: |
-      Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-fxdependent.zip" -Destination "$(Build.StagingDirectory)\symbols\fxdependent"
-    displayName: 'Expand symbols zip - fxdependent'
-
   - powershell: |
       tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi","$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" -path "$(System.ArtifactsDirectory)\package.xml"
     displayName: 'Generate Package Signing Xml'

tools/releaseBuild/build.json

@@ -15,7 +15,8 @@
             ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1"
         ],
         "DockerImageName": "ps-winsrvcore",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win7-x86",
@@ -32,7 +33,8 @@
             ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1"
         ],
         "DockerImageName": "ps-winsrvcore",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-x64-component-registration",
@@ -51,7 +53,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "componentregistration"
+        "VariableForExtractedBinariesPath": "componentregistration",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-x64-symbols",
@@ -70,7 +73,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "Symbols_x64"
+        "VariableForExtractedBinariesPath": "Symbols_x64",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-x86-symbols",
@@ -89,7 +93,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "Symbols_x86"
+        "VariableForExtractedBinariesPath": "Symbols_x86",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-arm-symbols",
@@ -108,7 +113,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "Symbols_arm"
+        "VariableForExtractedBinariesPath": "Symbols_arm",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-arm64-symbols",
@@ -127,7 +133,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "Symbols_arm64"
+        "VariableForExtractedBinariesPath": "Symbols_arm64",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-x64-package",
@@ -145,7 +152,8 @@
         ],
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "signed",
-        "ArtifactsExpected": 3
+        "ArtifactsExpected": 3,
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-x86-package",
@@ -163,7 +171,8 @@
         ],
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "signed",
-        "ArtifactsExpected": 3
+        "ArtifactsExpected": 3,
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-arm-package",
@@ -181,7 +190,8 @@
         ],
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "signed",
-        "ArtifactsExpected": 1
+        "ArtifactsExpected": 1,
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-arm64-package",
@@ -199,7 +209,8 @@
         ],
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "signed",
-        "ArtifactsExpected": 1
+        "ArtifactsExpected": 1,
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-fxdependent-symbols",
@@ -218,7 +229,8 @@
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "results",
         "ArtifactsExpected": 1,
-        "VariableForExtractedBinariesPath": "Symbols_fxdependent"
+        "VariableForExtractedBinariesPath": "Symbols_fxdependent",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "win-fxdependent-package",
@@ -236,7 +248,8 @@
         ],
         "DockerImageName": "ps-winsrvcore",
         "BinaryBucket": "signed",
-        "ArtifactsExpected": 1
+        "ArtifactsExpected": 1,
+        "EnableFeature": [ "ArtifactAsFolder" ]
     }
   ],
   "Linux": [
@@ -256,7 +269,8 @@
         "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_ubuntu14.04/Dockerfile",
         "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"],
         "DockerImageName": "ps-ubunutu-14-04",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "deb",
@@ -265,7 +279,8 @@
         "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_ubuntu18.04/Dockerfile",
         "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"],
         "DockerImageName": "ps-ubunutu-18-04",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "rpm",
@@ -274,7 +289,8 @@
         "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"],
         "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile",
         "DockerImageName": "ps-centos-7",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "alpine",
@@ -283,7 +299,8 @@
         "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"],
         "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile",
         "DockerImageName": "ps-alpine-3",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     },
     {
         "Name": "fxdependent",
@@ -292,7 +309,8 @@
         "AdditionalContextFiles" :[ "./tools/releaseBuild/Images/GenericLinuxFiles/PowerShellPackage.ps1"],
         "DockerFile": "./tools/releaseBuild/Images/microsoft_powershell_centos7/Dockerfile",
         "DockerImageName": "ps-centos-7",
-        "BinaryBucket": "release"
+        "BinaryBucket": "release",
+        "EnableFeature": [ "ArtifactAsFolder" ]
     }
   ]
 }

tools/releaseBuild/macOS/PowerShellPackageVsts.ps1

@@ -69,7 +69,7 @@ if ($Build.IsPresent) {
         $name = split-path -Leaf -Path $filePath
         $extension = (Split-Path -Extension -Path $filePath).Replace('.', '')
         Write-Verbose "Copying $filePath to $destination" -Verbose
-        Write-Host "##vso[artifact.upload containerfolder=results;artifactname=$name]$filePath"
+        Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$filePath"
         Write-Host "##vso[task.setvariable variable=Package-$extension]$filePath"
         Copy-Item -Path $filePath -Destination $destination -force
     }

tools/releaseBuild/vstsbuild.ps1

@@ -67,7 +67,8 @@ End {
         }
 
         $BuildPackagePath = New-PSSignedBuildZip -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath -DestinationFolder $destFolder
-        Write-Host "##vso[artifact.upload containerfolder=results;artifactname=$name-singed.zip]$BuildPackagePath"
+        Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath"
+        Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath"
         $buildPackageName = Split-Path -Path $BuildPackagePath -Leaf
         $additionalFiles += $BuildPackagePath
     }