diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 66d5c1aaa9..81c74e66d2 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -2,13 +2,6 @@ name: UnifiedPackageBuild-$(Build.BuildId) trigger: none parameters: - - name: ForceAzureBlobDelete - displayName: Delete Azure Blob - type: string - values: - - true - - false - default: false - name: InternalSDKBlobURL displayName: URL to the blob having internal .NET SDK type: string @@ -51,8 +44,6 @@ variables: value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - name: branchCounter value: $[counter(variables['branchCounterKey'], 1)] - - name: ForceAzureBlobDelete - value: ${{ parameters.ForceAzureBlobDelete }} - name: BUILDSECMON_OPT_IN value: true - name: __DOTNET_RUNTIME_FEED @@ -110,7 +101,37 @@ extends: stages: - stage: prep jobs: - - template: /.pipelines/templates/checkAzureContainer.yml@self + - job: SetVars + displayName: Set Variables + pool: + type: windows + + variables: + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem Env: + displayName: Capture environment variables + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no - stage: macos displayName: macOS - build and sign diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml index c91ceb9a9b..0dadf2f8f5 100644 --- a/.pipelines/PowerShell-Packages-Official.yml +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -1,6 +1,13 @@ trigger: none parameters: # parameters are shown up in ADO UI in a build queue time + - name: ForceAzureBlobDelete + displayName: Delete Azure Blob + type: string + values: + - true + - false + default: false - name: 'debug' displayName: 'Enable debug output' type: boolean @@ -27,6 +34,8 @@ variables: value: 1 - name: ROOT value: $(Build.SourcesDirectory) + - name: ForceAzureBlobDelete + value: ${{ parameters.ForceAzureBlobDelete }} - name: NUGET_XMLDOC_MODE value: none - name: nugetMultiFeedWarnLevel @@ -102,7 +111,12 @@ extends: enabled: false tsaOptionsFile: .config\tsaoptions.json stages: + - stage: prep + jobs: + - template: /.pipelines/templates/checkAzureContainer.yml@self + - stage: mac_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/mac-package-build.yml@self parameters: @@ -113,6 +127,7 @@ extends: buildArchitecture: arm64 - stage: windows_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/windows-package-build.yml@self parameters: @@ -139,6 +154,7 @@ extends: runtime: minsize - stage: linux_package + dependsOn: [prep] jobs: - template: /.pipelines/templates/linux-package-build.yml@self parameters: @@ -153,7 +169,7 @@ extends: signedDrop: 'drop_linux_sign_linux_fxd_x64_mariner' packageType: rpm-fxdependent #mariner-x64 jobName: mariner_x64 - signingProfile: 'CP-459159-Pgp' + signingProfile: 'CP-459159-pgpdetached' - template: /.pipelines/templates/linux-package-build.yml@self parameters: @@ -161,7 +177,7 @@ extends: signedDrop: 'drop_linux_sign_linux_fxd_arm64_mariner' packageType: rpm-fxdependent-arm64 #mariner-arm64 jobName: mariner_arm64 - signingProfile: 'CP-459159-Pgp' + signingProfile: 'CP-459159-pgpdetached' - template: /.pipelines/templates/linux-package-build.yml@self parameters: @@ -220,6 +236,7 @@ extends: jobName: minSize - stage: nupkg + dependsOn: [prep] jobs: - template: /.pipelines/templates/nupkg.yml@self diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index f5667c63c4..223517f2e9 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -17,6 +17,10 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: Skip Signing type: string default: 'NO' + - name: SkipPMCPublish + displayName: Skip PMC Publish + type: boolean + default: false variables: - name: CDP_DEFINITION_BUILD_COUNT @@ -270,7 +274,7 @@ extends: jobs: - template: /.pipelines/templates/release-publish-pmc.yml@self parameters: - skipPublish: true + skipPublish: ${{ parameters.SkipPMCPublish }} - stage: ReleaseDocker dependsOn: PublishGitHubRelease diff --git a/.pipelines/templates/SetVersionVariables.yml b/.pipelines/templates/SetVersionVariables.yml index 70aeff5b97..9894f9d53f 100644 --- a/.pipelines/templates/SetVersionVariables.yml +++ b/.pipelines/templates/SetVersionVariables.yml @@ -8,7 +8,7 @@ steps: - ${{ if eq(parameters['UseJson'],'yes') }}: - task: DownloadBuildArtifacts@0 inputs: - artifactName: 'drop_prep_DeleteBlob' + artifactName: 'drop_prep_SetVars' itemPattern: '*.json' downloadPath: '$(System.ArtifactsDirectory)' displayName: Download Build Info Json @@ -47,7 +47,18 @@ steps: - powershell: | $createJson = ("${{ parameters.CreateJson }}" -ne "no") - $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson + + $REPOROOT = $env:REPOROOT + + if (-not (Test-Path $REPOROOT/tools/releaseBuild/setReleaseTag.ps1)) { + if ((Test-Path "$REPOROOT/PowerShell/tools/releaseBuild/setReleaseTag.ps1")) { + $REPOROOT = "$REPOROOT/PowerShell" + } else { + throw "Could not find setReleaseTag.ps1 in $REPOROOT/tools/releaseBuild or $REPOROOT/PowerShell/tools/releaseBuild" + } + } + + $releaseTag = & "$REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson $version = $releaseTag.Substring(1) $vstsCommandString = "vso[task.setvariable variable=Version]$version" Write-Host ("sending " + $vstsCommandString) diff --git a/.pipelines/templates/checkAzureContainer.yml b/.pipelines/templates/checkAzureContainer.yml index d761d9f327..dae78a7e66 100644 --- a/.pipelines/templates/checkAzureContainer.yml +++ b/.pipelines/templates/checkAzureContainer.yml @@ -69,22 +69,27 @@ jobs: scriptType: inlineScript azurePowerShellVersion: latestVersion inline: | - try { - $container = Get-AzStorageContainer -Container '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -ErrorAction Stop - if ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'false') { - throw 'Azure blob container $(AzureVersion) already exists. To overwrite, use ForceAzureBlobDelete parameter' - } - elseif ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'true') { - Write-Verbose -Verbose 'Removing container $(AzureVersion) due to ForceAzureBlobDelete parameter' - Remove-AzStorageContainer -Name '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -Force - } - } - catch { - if ($_.FullyQualifiedErrorId -eq 'ResourceNotFoundException,Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet.GetAzureStorageContainerCommand') { - Write-Verbose -Verbose 'Container "$(AzureVersion)" does not exists.' - } - else { - throw $_ - } + $containersToDelete = @('$(AzureVersion)', '$(AzureVersion)-private', '$(AzureVersion)-nuget', '$(AzureVersion)-gc') + + $containersToDelete | ForEach-Object { + $containerName = $_ + try { + $container = Get-AzStorageContainer -Container $containerName -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -ErrorAction Stop + if ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'false') { + throw "Azure blob container $containerName already exists. To overwrite, use ForceAzureBlobDelete parameter" + } + elseif ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'true') { + Write-Verbose -Verbose "Removing container $containerName due to ForceAzureBlobDelete parameter" + Remove-AzStorageContainer -Name $containerName -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -Force + } + } + catch { + if ($_.FullyQualifiedErrorId -eq 'ResourceNotFoundException,Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet.GetAzureStorageContainerCommand') { + Write-Verbose -Verbose "Container $containerName does not exists." + } + else { + throw $_ + } + } } - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/linux-package-build.yml b/.pipelines/templates/linux-package-build.yml index 34b85bbdb0..28df10de5a 100644 --- a/.pipelines/templates/linux-package-build.yml +++ b/.pipelines/templates/linux-package-build.yml @@ -3,6 +3,7 @@ parameters: signedeDrop: 'drop_linux_sign_linux_x64' packageType: deb jobName: 'deb' + signingProfile: 'CP-450779-pgpdetached' jobs: - job: ${{ parameters.jobName }} diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index 9d2c47a20a..2e6f9c3e5e 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -77,7 +77,14 @@ jobs: Start-PSBootstrap $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose - Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath @params -Clean -PSModuleRestore + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath @params -Clean -PSModuleRestore @ReleaseTagParam $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' $null = New-Item -ItemType Directory -Path $outputPath -Force diff --git a/.pipelines/templates/mac.yml b/.pipelines/templates/mac.yml index dfb986f45a..b71e83dd81 100644 --- a/.pipelines/templates/mac.yml +++ b/.pipelines/templates/mac.yml @@ -56,7 +56,14 @@ jobs: # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. Import-Module ./build.psm1 -Force - Start-PSBuild -Runtime 'osx-${{ parameters.buildArchitecture }}' -Configuration Release -PSModuleRestore -Clean -Output $(OB_OUTPUTDIRECTORY) + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime 'osx-${{ parameters.buildArchitecture }}' -Configuration Release -PSModuleRestore -Clean -Output $(OB_OUTPUTDIRECTORY) @ReleaseTagParam $artifactName = "macosBinResults-${{ parameters.buildArchitecture }}" $psOptPath = "$(OB_OUTPUTDIRECTORY)/psoptions.json" diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml index 3b10b26958..11b4573327 100644 --- a/.pipelines/templates/release-MakeBlobPublic.yml +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -46,8 +46,6 @@ jobs: CreateJson: yes UseJson: no - - template: /.pipelines/templates/cloneToOfficialPath.yml@self - - pwsh: | Get-ChildItem Env: displayName: 'Capture Environment Variables' @@ -63,27 +61,15 @@ jobs: Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose displayName: Remove AzRM modules - - task: AzurePowerShell@5 - displayName: Set blob permissions + - task: AzureCLI@2 + displayName: 'Set blob permissions' inputs: azureSubscription: az-blob-cicd-infra - scriptType: inlineScript - azurePowerShellVersion: LatestVersion - pwsh: true - inline: | - $storageAccount = "$(StorageAccount)" - $containerName = "$(azureVersion)" - $publicAccess = "blob" - - Set-AzStorageContainerAcl ` - -Context (New-AzStorageContext -StorageAccountName $storageAccount) ` - -Container $containerName ` - -PublicAccess $publicAccess - - Set-AzStorageContainerAcl ` - -Context (New-AzStorageContext -StorageAccountName $storageAccount) ` - -Container "$containerName-gc" ` - -PublicAccess $publicAccess + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob + az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion)-gc --public-access blob - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -104,6 +90,10 @@ jobs: - group: 'Azure Blob variable group' - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - checkout: self @@ -117,8 +107,6 @@ jobs: CreateJson: yes UseJson: no - - template: /.pipelines/templates/cloneToOfficialPath.yml@self - - pwsh: | Get-ChildItem Env: displayName: 'Capture Environment Variables' @@ -143,28 +131,31 @@ jobs: pwsh: true inline: | $sourceStorageAccountName = '$(StorageAccount)' - $sourceContainerName = '$(AzureVersion)' + $sourceContainerName = '$(AzureVersion)-nuget' $destinationStorageAccountName = '$(PSInfraStorageAccount)' - $destinationContainerName = "tool/$(Version)" + $destinationContainerName = "tool" $sourceContext = New-AzStorageContext -StorageAccountName $sourceStorageAccountName - $destinationContext = New-AzStorageContext -StorageAccountName $destinationStorageAccountName + Write-Verbose -Verbose "Source context: $($sourceContext.BlobEndPoint)" - $blobs = Get-AzStorageBlob -Context $sourceContext -Container $sourceContainerName + $destinationContext = New-AzStorageContext -StorageAccountName $destinationStorageAccountName + Write-Verbose -Verbose "Destination context: $($destinationContext.BlobEndPoint)" + + $prefix = 'globaltool' + $blobs = Get-AzStorageBlob -Context $sourceContext -Container $sourceContainerName -Prefix $prefix + + Write-Verbose -Verbose "Blobs found in $sourceContainerName" + $blobs.Name | Write-Verbose -Verbose + + Write-Verbose -Verbose "Copying blobs from $sourceContainerName to $destinationContainerName" foreach ($blob in $blobs) { $sourceBlobName = $blob.Name - $destinationBlobName = $sourceBlobName + Write-Verbose -Verbose "sourceBlobName = $sourceBlobName" - $destinationBlob = Start-AzStorageBlobCopy -Context $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName + $destinationBlobName = $sourceBlobName -replace "$prefix", '$(Version)' + Write-Verbose -Verbose "destinationBlobName = $destinationBlobName" - # Wait for the copy operation to complete - $destinationBlob | Wait-AzStorageBlobCopy - - if ($destinationBlob.CopyStatus -eq "Success") { - Write-Host "Blob copy completed successfully for $sourceBlobName." - } else { - Write-Host "Blob copy failed for $sourceBlobName." - } + Copy-AzStorageBlob -SourceContext $sourceContext -DestinationContext $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName -Force -Verbose -Confirm:$false } diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index da5e5136a5..89f2e7b5a2 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -76,6 +76,18 @@ jobs: displayName: Create MsixBundle retryCountOnTaskFailure: 1 + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + + displayName: Remove AzRM modules and install Az.Storage + - task: AzurePowerShell@5 displayName: Upload msix to blob inputs: @@ -90,10 +102,10 @@ jobs: $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount if ($env:BundleDir) { - $bundleFile = "$env:BundleDir\*.msixbundle" - $blobName = Get-Item $bundleFile | Split-Path -Leaf + $bundleFile = Get-Item "$env:BundleDir\*.msixbundle" + $blobName = $bundleFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $bundleFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + Set-AzStorageBlobContent -File $bundleFile -Container $containerName -Blob $blobName -Context $storageContext } else{ throw "BundleDir not found" diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index e94fa16fef..c1e5d1a06a 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -73,6 +73,10 @@ jobs: Write-Output "Downloaded $blobName to $destinationFile" } + $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName + Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" + Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" + - pwsh: | Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty FullName displayName: Capture downloaded artifacts diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index c94efc52e1..e9bebf5d93 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -47,10 +47,10 @@ jobs: - pwsh: | #Exclude all global tool packages. Their names start with 'PowerShell.' $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" - Copy-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose + Copy-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose $releaseVersion = '$(VERSION)' - $globalToolPath = "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/PowerShell.$releaseVersion.nupkg" + $globalToolPath = "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/PowerShell.$releaseVersion.nupkg" if ($releaseVersion -notlike '*-*') { # Copy the global tool package for stable releases @@ -59,11 +59,11 @@ jobs: Get-ChildItem "$(Pipeline.Workspace)/release" -recurse displayName: Download and capture nupkgs - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) + condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - task: NuGetCommand@2 displayName: 'NuGet push' - condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) + condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) inputs: command: push packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 0d752b09e1..93032f35b3 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -9,6 +9,10 @@ jobs: condition: succeeded() pool: type: linux + isCustom: true + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure variables: - name: runCodesignValidationInjection value: false @@ -30,6 +34,9 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: + - checkout: self ## the global setting on lfs didn't work + lfs: false + - template: release-SetReleaseTagAndContainerName.yml - pwsh: | @@ -40,7 +47,7 @@ jobs: displayName: Set Package version - pwsh: | - $branch = 'main-mirror' + $branch = 'mirror-target' $gitArgs = "clone", "--verbose", "--branch", @@ -57,7 +64,7 @@ jobs: pythonDownloadServiceConnections: pmcDownload - pwsh: | - pip install pmc-cli + pip install pmc-cli==1.12.0 $newPath = (resolve-path '~/.local/bin').providerpath $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" diff --git a/.pipelines/templates/release-symbols.yml b/.pipelines/templates/release-symbols.yml index ec8e91d2fb..f2260d57a8 100644 --- a/.pipelines/templates/release-symbols.yml +++ b/.pipelines/templates/release-symbols.yml @@ -51,17 +51,18 @@ jobs: - pwsh: | Write-Verbose -Verbose "Enumerating $(Pipeline.Workspace)\CoOrdinatedBuildPipeline" - $downloadedArtifacts = Get-ChildItem -Recurse "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline" -Recurse -Filter 'symbols.zip' + $downloadedArtifacts = Get-ChildItem -Path "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline" -Recurse -Filter 'symbols.zip' $downloadedArtifacts $expandedRoot = New-Item -Path "$(Pipeline.Workspace)/expanded" -ItemType Directory -Verbose $symbolsRoot = New-Item -Path "$(Pipeline.Workspace)/symbols" -ItemType Directory -Verbose $downloadedArtifacts | ForEach-Object { $folderName = (Get-Item (Split-Path $_.FullName)).Name + Write-Verbose -Verbose "Expanding $($_.FullName) to $expandedRoot/$folderName/$($_.BaseName)" $destFolder = New-Item -Path "$expandedRoot/$folderName/$($_.BaseName)/" -ItemType Directory -Verbose Expand-Archive -Path $_.FullName -DestinationPath $destFolder -Force - $symbolsToPublish = New-Item -Path "$symbolsRoot/$($_.BaseName)" -ItemType Directory -Verbose + $symbolsToPublish = New-Item -Path "$symbolsRoot/$folderName/$($_.BaseName)" -ItemType Directory -Verbose Get-ChildItem -Path $destFolder -Recurse -Filter '*.pdb' | ForEach-Object { Copy-Item -Path $_.FullName -Destination $symbolsToPublish -Verbose diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml index b13c1ee73f..bf3abcc3c5 100644 --- a/.pipelines/templates/release-upload-buildinfo.yml +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -4,8 +4,8 @@ parameters: type: boolean jobs: -- job: PMCPublish - displayName: Publish to PMC +- job: BuildInfoPublish + displayName: Publish BuildInfo condition: succeeded() pool: type: windows @@ -18,6 +18,7 @@ jobs: value: 1 - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - group: 'Azure Blob variable group' - name: ob_sdl_codeSignValidation_enabled value: false - name: ob_sdl_binskim_enabled @@ -93,6 +94,17 @@ jobs: } displayName: Create json files + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + displayName: Remove AzRM modules + - task: AzurePowerShell@5 displayName: Upload buildjson to blob inputs: @@ -101,7 +113,7 @@ jobs: azurePowerShellVersion: LatestVersion pwsh: true inline: | - $containerName = "BuildInfo" + $containerName = "buildinfo" $storageAccount = '$(StorageAccount)' $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount @@ -110,20 +122,20 @@ jobs: $jsonFile = "$env:BuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force } if ($env:CopyLTSBuildInfo -eq 'YES') { $jsonFile = "$env:LtsBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force } if ($env:CopyVersionBuildInfo -eq 'YES') { $jsonFile = "$env:VersionBuildInfoJsonFile" $blobName = Get-Item $jsonFile | Split-Path -Leaf Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" - Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force } condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES')) diff --git a/.pipelines/templates/release-validate-packagenames.yml b/.pipelines/templates/release-validate-packagenames.yml index 564b9d7c78..cadf0c1ba1 100644 --- a/.pipelines/templates/release-validate-packagenames.yml +++ b/.pipelines/templates/release-validate-packagenames.yml @@ -13,12 +13,15 @@ jobs: - group: 'Azure Blob variable group' steps: - - pwsh: | - Get-ChildItem ENV: - displayName: Capture environment + - checkout: self + clean: true - template: release-SetReleaseTagAndContainerName.yml + - pwsh: | + Get-ChildItem ENV: + displayName: Capture environment + - pwsh: | $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks Write-Host $name diff --git a/.pipelines/templates/testartifacts.yml b/.pipelines/templates/testartifacts.yml index 883d3cd21c..d3e2f25488 100644 --- a/.pipelines/templates/testartifacts.yml +++ b/.pipelines/templates/testartifacts.yml @@ -12,6 +12,9 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_codeSignValidation_excludes + value: '-|**\*.ps1;-|**\*.psm1;-|**\*.ps1xml;-|**\*.psd1;-|**\*.exe;-|**\*.dll;-|**\*.cdxml' + displayName: Build windows test artifacts condition: succeeded() pool: @@ -29,6 +32,7 @@ jobs: env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - pwsh: | + New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 function BuildTestPackage([string] $runtime) { @@ -45,13 +49,16 @@ jobs: win-arm64 { $packageName = "TestPackage-win-arm64.zip" } } Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName - Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" + ## Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" + + Copy-Item -Path $(System.ArtifactsDirectory)/$packageName -Destination $(ob_outputDirectory) -Force -Verbose } BuildTestPackage -runtime win7-x64 BuildTestPackage -runtime win7-x86 BuildTestPackage -runtime win-arm64 displayName: Build test package and upload retryCountOnTaskFailure: 1 + - job: build_testartifacts_nonwin variables: - name: runCodesignValidationInjection @@ -78,6 +85,7 @@ jobs: env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - pwsh: | + New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 function BuildTestPackage([string] $runtime) { @@ -96,7 +104,7 @@ jobs: linux-musl-x64 { $packageName = "TestPackage-alpine-x64.zip"} } Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName - Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" + Copy-Item -Path $(System.ArtifactsDirectory)/$packageName -Destination $(ob_outputDirectory) -Force -Verbose } BuildTestPackage -runtime linux-x64 BuildTestPackage -runtime linux-arm @@ -105,4 +113,3 @@ jobs: BuildTestPackage -runtime linux-musl-x64 displayName: Build test package and upload retryCountOnTaskFailure: 1 - - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 3860b1a811..6ecedc30f7 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -88,7 +88,14 @@ jobs: Start-PSBootstrap -Package $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose - Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params @ReleaseTagParam $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' Write-Verbose -Verbose "refFolderPath: $refFolderPath" @@ -111,45 +118,71 @@ jobs: $pdbs | Compress-Archive -DestinationPath "$(ob_outputDirectory)/symbols.zip" -Update } - if ($runtime -eq 'fxdependent') - { - ## Also build global tool - Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose - $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' - Push-Location -Path $globalToolCsProjDir -Verbose - - $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' - $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) - $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' - Pop-Location - # do this to ensure everything gets signed. - Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath - - # Copy reference assemblies - Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force - - Write-Verbose -Verbose "clean unnecessary files in obj directory" - $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' - - $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") - - # only four files are needed in obj folder for global tool packaging - Get-ChildItem -Path $objDir -File -Recurse | - Where-Object { -not $_.PSIsContainer } | - Where-Object { $_.name -notin $filesToKeep } | - Remove-Item -Verbose - } - Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols folder' env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + - pwsh: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + Start-PSBootstrap + + ## Build global tool + Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' + $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + if ($env:RELEASETAGVAR) { + $ReleaseTagToUse = $env:RELEASETAGVAR -Replace '^v' + } + + Write-Verbose -Verbose "Building PowerShell global tool for Windows.x64 with cmdline: dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse" + dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse + $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' + Pop-Location + # do this to ensure everything gets signed. + Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath + + $buildWithSymbolsPath = Get-Item -Path "$(Pipeline.Workspace)/Symbols_$(Architecture)" + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + + # Copy reference assemblies + Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force + + Write-Verbose -Verbose "clean unnecessary files in obj directory" + $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' + + $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") + + # only four files are needed in obj folder for global tool packaging + Get-ChildItem -Path $objDir -File -Recurse | + Where-Object { -not $_.PSIsContainer } | + Where-Object { $_.name -notin $filesToKeep } | + Remove-Item -Verbose + + + displayName: 'Build Winx64 Global tool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') env: @@ -205,13 +238,13 @@ jobs: $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' Push-Location -Path $globalToolCsProjDir -Verbose - <# - $nuspecFilePath = "$globalToolCsProjDir\PowerShell.Windows.x64.nuspec" - $nuSpec = $packagingStrings.WindowsX64GlobalToolNuspec -f '$(Version)' - $nuSpec | Out-File -FilePath $nuspecFilePath -Encoding ascii - #> + if ($env:RELASETAGVAR) { + $ReleaseTagToUse = $env:RELASETAGVAR -Replace '^v' + } - dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png + Write-Verbose -Verbose "Packing PowerShell global tool for Windows.x64 with cmdline: dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse" + + dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse Write-Verbose -Verbose "Deleting content and contentFiles folders from the nupkg" diff --git a/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj index 045faad614..f18bdae611 100644 --- a/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj +++ b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj @@ -9,6 +9,8 @@ win-x64 pwsh $(PackageVersion) + true + ../../signing/visualstudiopublic.snk