From 158ce7062359feac6192c4920cc84cff4aa8ea00 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 20 May 2024 21:54:00 +0000 Subject: [PATCH] Merged PR 31196: Fix release version and stage issues in build and packaging Fix release version and stage issues in build and packaging --- ...werShell-Coordinated_Packages-Official.yml | 41 +++++-- .pipelines/PowerShell-Packages-Official.yml | 8 ++ .pipelines/templates/SetVersionVariables.yml | 2 +- .pipelines/templates/release-create-msix.yml | 12 ++ .../release-validate-packagenames.yml | 9 +- .pipelines/templates/windows-hosted-build.yml | 104 +++++++++++------- .../PowerShell.Windows.x64.csproj | 2 + 7 files changed, 125 insertions(+), 53 deletions(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 66d5c1aaa9..81c74e66d2 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -2,13 +2,6 @@ name: UnifiedPackageBuild-$(Build.BuildId) trigger: none parameters: - - name: ForceAzureBlobDelete - displayName: Delete Azure Blob - type: string - values: - - true - - false - default: false - name: InternalSDKBlobURL displayName: URL to the blob having internal .NET SDK type: string @@ -51,8 +44,6 @@ variables: value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - name: branchCounter value: $[counter(variables['branchCounterKey'], 1)] - - name: ForceAzureBlobDelete - value: ${{ parameters.ForceAzureBlobDelete }} - name: BUILDSECMON_OPT_IN value: true - name: __DOTNET_RUNTIME_FEED @@ -110,7 +101,37 @@ extends: stages: - stage: prep jobs: - - template: /.pipelines/templates/checkAzureContainer.yml@self + - job: SetVars + displayName: Set Variables + pool: + type: windows + + variables: + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem Env: + displayName: Capture environment variables + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no - stage: macos displayName: macOS - build and sign diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml index c91ceb9a9b..688cbe4775 100644 --- a/.pipelines/PowerShell-Packages-Official.yml +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -102,7 +102,12 @@ extends: enabled: false tsaOptionsFile: .config\tsaoptions.json stages: + - stage: prep + jobs: + - template: /.pipelines/templates/checkAzureContainer.yml@self + - stage: mac_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/mac-package-build.yml@self parameters: @@ -113,6 +118,7 @@ extends: buildArchitecture: arm64 - stage: windows_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/windows-package-build.yml@self parameters: @@ -139,6 +145,7 @@ extends: runtime: minsize - stage: linux_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/linux-package-build.yml@self parameters: @@ -220,6 +227,7 @@ extends: jobName: minSize - stage: nupkg + dependsOn: [prep] jobs: - template: /.pipelines/templates/nupkg.yml@self diff --git a/.pipelines/templates/SetVersionVariables.yml b/.pipelines/templates/SetVersionVariables.yml index 70aeff5b97..6719abf6da 100644 --- a/.pipelines/templates/SetVersionVariables.yml +++ b/.pipelines/templates/SetVersionVariables.yml @@ -8,7 +8,7 @@ steps: - ${{ if eq(parameters['UseJson'],'yes') }}: - task: DownloadBuildArtifacts@0 inputs: - artifactName: 'drop_prep_DeleteBlob' + artifactName: 'drop_prep_SetVars' itemPattern: '*.json' downloadPath: '$(System.ArtifactsDirectory)' displayName: Download Build Info Json diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index da5e5136a5..fb427a760b 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -76,6 +76,18 @@ jobs: displayName: Create MsixBundle retryCountOnTaskFailure: 1 + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + + displayName: Remove AzRM modules and install Az.Storage + - task: AzurePowerShell@5 displayName: Upload msix to blob inputs: diff --git a/.pipelines/templates/release-validate-packagenames.yml b/.pipelines/templates/release-validate-packagenames.yml index 564b9d7c78..b484433486 100644 --- a/.pipelines/templates/release-validate-packagenames.yml +++ b/.pipelines/templates/release-validate-packagenames.yml @@ -13,12 +13,15 @@ jobs: - group: 'Azure Blob variable group' steps: - - pwsh: | - Get-ChildItem ENV: - displayName: Capture environment + - checkout: self + clean: true - template: release-SetReleaseTagAndContainerName.yml + - pwsh: | + Get-ChildItem ENV: + displayName: Capture environment + - pwsh: | $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks Write-Host $name diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 81f5258a68..9e0a18ec23 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -117,45 +117,71 @@ jobs: $pdbs | Compress-Archive -DestinationPath "$(ob_outputDirectory)/symbols.zip" -Update } - if ($runtime -eq 'fxdependent') - { - ## Also build global tool - Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose - $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' - Push-Location -Path $globalToolCsProjDir -Verbose - - $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' - $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) - $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' - Pop-Location - # do this to ensure everything gets signed. - Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath - - # Copy reference assemblies - Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force - - Write-Verbose -Verbose "clean unnecessary files in obj directory" - $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' - - $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") - - # only four files are needed in obj folder for global tool packaging - Get-ChildItem -Path $objDir -File -Recurse | - Where-Object { -not $_.PSIsContainer } | - Where-Object { $_.name -notin $filesToKeep } | - Remove-Item -Verbose - } - Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols folder' env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + - pwsh: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + Start-PSBootstrap + + ## Build global tool + Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' + $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + if ($env:RELEASETAGVAR) { + $ReleaseTagToUse = $env:RELEASETAGVAR -Replace '^v' + } + + Write-Verbose -Verbose "Building PowerShell global tool for Windows.x64 with cmdline: dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse" + dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse + $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' + Pop-Location + # do this to ensure everything gets signed. + Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath + + $buildWithSymbolsPath = Get-Item -Path "$(Pipeline.Workspace)/Symbols_$(Architecture)" + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + + # Copy reference assemblies + Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force + + Write-Verbose -Verbose "clean unnecessary files in obj directory" + $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' + + $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") + + # only four files are needed in obj folder for global tool packaging + Get-ChildItem -Path $objDir -File -Recurse | + Where-Object { -not $_.PSIsContainer } | + Where-Object { $_.name -notin $filesToKeep } | + Remove-Item -Verbose + + + displayName: 'Build Winx64 Global tool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') env: @@ -211,13 +237,13 @@ jobs: $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' Push-Location -Path $globalToolCsProjDir -Verbose - <# - $nuspecFilePath = "$globalToolCsProjDir\PowerShell.Windows.x64.nuspec" - $nuSpec = $packagingStrings.WindowsX64GlobalToolNuspec -f '$(Version)' - $nuSpec | Out-File -FilePath $nuspecFilePath -Encoding ascii - #> + if ($env:RELASETAGVAR) { + $ReleaseTagToUse = $env:RELASETAGVAR -Replace '^v' + } - dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png + Write-Verbose -Verbose "Packing PowerShell global tool for Windows.x64 with cmdline: dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse" + + dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse Write-Verbose -Verbose "Deleting content and contentFiles folders from the nupkg" diff --git a/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj index 045faad614..f18bdae611 100644 --- a/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj +++ b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj @@ -9,6 +9,8 @@ win-x64 pwsh $(PackageVersion) + true + ../../signing/visualstudiopublic.snk