From 72d29a14d57eed258e13d4e1d10f98e613f11833 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 10 Nov 2023 18:55:02 +0000 Subject: [PATCH] Merged PR 28438: [backport/v7.4.0] Copy azure blob with PowerShell global tool to private blob and move to CDN d... Copy azure blob with PowerShell global tool to private blob and move to CDN during release (#20659) --- .../azureDevOps/releasePipeline.yml | 60 ++++++++++++++----- .../azureDevOps/templates/nuget.yml | 2 +- .../templates/release-CopyGlobalTools.yml | 56 +++++++++++++++++ .../release-SetReleaseTagAndContainerName.yml | 5 ++ 4 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index fb6b8d5d95..05dbe1799e 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -290,8 +290,8 @@ stages: Update and merge the changelog for the release. This step is required for creating GitHub draft release. -- stage: GitHubDraftRelease - displayName: Create GitHub draft release +- stage: BlobPublic + displayName: Make Blob Public # do not include stages that are likely to fail in dependency as there is no way to force deploy. dependsOn: UpdateChangeLog @@ -314,6 +314,38 @@ stages: steps: - template: templates/release-MakeContainerPublic.yml + - template: templates/release/approvalJob.yml + parameters: + displayName: Copy Global tool packages to PSInfra storage + jobName: CopyBlobApproval + instructions: | + Approval for Copy global tool packages to PSInfra storage + + - job: PSInfraBlobPublic + displayName: Copy global tools to PSInfra storage + dependsOn: CopyBlobApproval + + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + + variables: + - group: 'PSInfraStorage' + + steps: + - template: templates/release-CopyGlobalTools.yml + parameters: + sourceContainerName: 'tool-private' + destinationContainerName: 'tool' + sourceStorageAccountName: '$(GlobalToolStorageAccount)' + destinationStorageAccountName: '$(PSInfraStorageAccount)' + blobPrefix: '$(Version)' + +- stage: GitHubTasks + displayName: GitHub tasks + dependsOn: BlobPublic + jobs: - job: GitHubDraft displayName: Create GitHub Draft release @@ -326,28 +358,24 @@ stages: - group: 'Azure Blob variable group' - group: 'AzDevOpsArtifacts' - group: ReleasePipelineSecrets - dependsOn: AzureBlobPublic steps: - template: templates/release-CreateGitHubDraft.yml -- stage: GitHubManualTasks - displayName: GitHub manual tasks - dependsOn: GitHubDraftRelease - jobs: - deployment: PushTag + dependsOn: GitHubDraft displayName: Push Git Tag pool : server environment: PSReleasePushTag - deployment: MakeDraftPublic + dependsOn: PushTag displayName: Make GitHub Draft public pool : server environment: PSReleaseDraftPublic - dependsOn: PushTag - stage: PublishPackages displayName: Publish packages - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks jobs: - job: PublishNuget @@ -405,7 +433,7 @@ stages: - stage: ReleaseDocker displayName: Release Docker dependsOn: - - GitHubManualTasks + - GitHubTasks jobs: - deployment: ReleaseDocker displayName: Release Docker @@ -504,7 +532,7 @@ stages: Notify the PM team to start the process of releasing to MU. - stage: UpdateDotnetDocker - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Update DotNet SDK Docker images jobs: - template: templates/release/approvalJob.yml @@ -519,7 +547,7 @@ stages: 4. create PR targeting nightly branch - stage: UpdateWinGet - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Add manifest entry to winget jobs: - template: templates/release/approvalJob.yml @@ -530,7 +558,7 @@ stages: This is typically done by the community 1-2 days after the release. - stage: PublishMsix - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Publish MSIX to store jobs: - template: templates/release/approvalJob.yml @@ -541,7 +569,7 @@ stages: Ask Steve to release MSIX bundle package to Store - stage: BuildInfoJson - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Upload BuildInfoJson jobs: - deployment: UploadJson @@ -562,7 +590,7 @@ stages: - template: templates/release-BuildJson.yml - stage: ReleaseVPack - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Release VPack jobs: - job: KickoffvPack @@ -611,7 +639,7 @@ stages: } - stage: ReleaseDeps - dependsOn: GitHubManualTasks + dependsOn: GitHubTasks displayName: Update pwsh.deps.json links jobs: - template: templates/release-UpdateDepsJson.yml diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 702e33b7e8..22f791bf0e 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -279,7 +279,7 @@ jobs: azureSubscription: '$(GlobalToolSubscription)' Destination: AzureBlob storage: '$(GlobalToolStorageAccount)' - ContainerName: 'tool' + ContainerName: 'tool-private' blobPrefix: '$(Version)' condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) retryCountOnTaskFailure: 2 diff --git a/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml b/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml new file mode 100644 index 0000000000..7c9306496e --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/release-CopyGlobalTools.yml @@ -0,0 +1,56 @@ +parameters: +- name: sourceContainerName + type: string + default: 'source-container' + +- name: destinationContainerName + type: string + default: 'destination-container' + +- name: sourceStorageAccountName + type: string + default: 'source-storage-account' + +- name: destinationStorageAccountName + type: string + default: 'destination-storage-account' + +- name: blobPrefix + type: string + default: '$(Version)' + +steps: +- template: release-SetReleaseTagAndContainerName.yml + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + retryCountOnTaskFailure: 2 + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy + Write-Verbose -Verbose "Found AzCopy: $azcopy" + + $sourceContainerName = "${{ parameters.sourceContainerName }}" + $destinationContainerName = "${{ parameters.destinationContainerName }}" + $sourceStorageAccountName = "${{ parameters.sourceStorageAccountName }}" + $destinationStorageAccountName = "${{ parameters.destinationStorageAccountName }}" + $blobPrefix = "${{ parameters.blobPrefix }}" + + $sourceBlobUrl = "https://${sourceStorageAccountName}.blob.core.windows.net/${sourceContainerName}/${blobPrefix}" + Write-Verbose -Verbose "Source blob url: $sourceBlobUrl" + $destinationBlobUrl = "https://${destinationStorageAccountName}.blob.core.windows.net/${destinationContainerName}" + Write-Verbose -Verbose "Destination blob url: $destinationBlobUrl" + + & $azcopy cp $sourceBlobUrl $destinationBlobUrl --recursive + + $packagesPath = Get-ChildItem -Path $(System.ArtifactsDirectory)\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName + Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" + Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" + + displayName: Copy blobs + retryCountOnTaskFailure: 2 + env: + AZCOPY_AUTO_LOGIN_TYPE: MSI diff --git a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml b/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml index 26229325b8..7e88624b45 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml @@ -18,4 +18,9 @@ steps: $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" + + $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" displayName: Set container name