diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 77ac93401d..66d5c1aaa9 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -93,7 +93,7 @@ extends: cg: enabled: true ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' - asyncSdl: # https://aka.ms/obpipelines/asyncsdl + asyncSdl: enabled: true forStages: [prep, macos, linux, windows, SignFiles, test_and_release_artifacts] credscan: diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml index b7fbee78cb..c91ceb9a9b 100644 --- a/.pipelines/PowerShell-Packages-Official.yml +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -1,4 +1,4 @@ -trigger: none # https://aka.ms/obpipelines/triggers +trigger: none parameters: # parameters are shown up in ADO UI in a build queue time - name: 'debug' @@ -20,7 +20,7 @@ parameters: # parameters are shown up in ADO UI in a build queue time variables: - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning + value: $[counter('', 0)] # needed for onebranch.pipeline.version task - name: system.debug value: ${{ parameters.debug }} - name: ENABLE_PRS_DELAYSIGN @@ -40,7 +40,7 @@ variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project - name: LinuxContainerImage value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 - group: mscodehub-feed-read-akv @@ -66,9 +66,9 @@ resources: ref: refs/heads/main extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: - cloudvault: # https://aka.ms/obpipelines/cloudvault + cloudvault: enabled: false featureFlags: linuxEsrpSigning: true @@ -88,7 +88,7 @@ extends: cg: enabled: true ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' - asyncSdl: # https://aka.ms/obpipelines/asyncsdl + asyncSdl: enabled: true forStages: ['build'] credscan: diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml new file mode 100644 index 0000000000..f5667c63c4 --- /dev/null +++ b/.pipelines/PowerShell-Release-Official.yml @@ -0,0 +1,409 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: REPOROOT + value: $(Build.SourcesDirectory) + - name: OUTPUTROOT + value: $(REPOROOT)\out + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Packages-Official' + + - pipeline: PSPackagesOfficial + source: 'PowerShell-Packages-Official' + trigger: + branches: + include: + - master + - releases/* + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates + parameters: + cloudvault: + enabled: false + globalSdl: + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + asyncSdl: + enabled: true + tsaOptionsFile: .config/tsaoptions.json + tsa: + enabled: true + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: false # always break the build on binskim issues in addition to TSA upload + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # suppression: + # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + tsaOptionsFile: .config\tsaoptions.json + + stages: + - stage: msixbundle + displayName: 'Create MSIX Bundle' + jobs: + - template: /.pipelines/templates/release-create-msix.yml@self + + - stage: validateSdk + displayName: 'Validate SDK' + jobs: + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "windowsSDK" + displayName: "Windows SDK Validation" + jobtype: windows + + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "MacOSSDK" + displayName: "MacOS SDK Validation" + jobtype: macos + + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "LinuxSDK" + displayName: "Linux SDK Validation" + jobtype: linux + + - stage: gbltool + displayName: 'Validate Global tools' + jobs: + - template: /.pipelines/templates/release-validate-globaltools.yml@self + parameters: + jobName: "WindowsGlobalTools" + displayName: "Windows Global Tools Validation" + jobtype: windows + + - template: /.pipelines/templates/release-validate-globaltools.yml@self + parameters: + jobName: "LinuxGlobalTools" + displayName: "Linux Global Tools Validation" + jobtype: linux + globalToolExeName: 'pwsh' + globalToolPackageName: 'PowerShell.Linux.x64' + + - stage: fxdpackages + displayName: 'Validate FXD Packages' + jobs: + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'winfxd' + displayName: 'Validate Win Fxd Packages' + jobtype: 'windows' + artifactName: 'drop_windows_package_package_win_fxdependent' + packageNamePattern: '**/*win-fxdependent.zip' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'winfxdDesktop' + displayName: 'Validate WinDesktop Fxd Packages' + jobtype: 'windows' + artifactName: 'drop_windows_package_package_win_fxdependentWinDesktop' + packageNamePattern: '**/*win-fxdependentwinDesktop.zip' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'linuxfxd' + displayName: 'Validate Linux Fxd Packages' + jobtype: 'linux' + artifactName: 'drop_linux_package_fxdependent' + packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'linuxArm64fxd' + displayName: 'Validate Linux ARM64 Fxd Packages' + jobtype: 'linux' + artifactName: 'drop_linux_package_fxdependent' + packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' + arm64: 'yes' + + - stage: validatePackages + displayName: 'Validate Packages' + jobs: + - template: /.pipelines/templates/release-validate-packagenames.yml@self + + - stage: ManualValidation + dependsOn: [] + displayName: Manual Validation + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Validate Windows Packages + jobName: ValidateWinPkg + instructions: | + Validate zip package on windows + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Validate OSX Packages + jobName: ValidateOsxPkg + instructions: | + Validate tar.gz package on osx-arm64 + + - stage: ReleaseAutomation + dependsOn: [] + displayName: 'Release Automation' + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start Release Automation + jobName: StartRA + instructions: | + Kick off Release automation build at: https://dev.azure.com/powershell-rel/Release-Automation/_build?definitionId=10&_a=summary + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Triage results + jobName: TriageRA + dependsOnJob: StartRA + instructions: | + Triage ReleaseAutomation results + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Signoff Tests + dependsOnJob: TriageRA + jobName: SignoffTests + instructions: | + Signoff ReleaseAutomation results + + - stage: UpdateChangeLog + displayName: Update the changelog + dependsOn: + - ManualValidation + - ReleaseAutomation + - validatePackages + - fxdpackages + - gbltool + - validateSdk + - msixbundle + + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make sure the changelog is updated + jobName: MergeChangeLog + instructions: | + Update and merge the changelog for the release. + This step is required for creating GitHub draft release. + + - stage: BlobPublic + displayName: Make Blob Public + dependsOn: UpdateChangeLog + jobs: + - template: /.pipelines/templates/release-MakeBlobPublic.yml@self + + - stage: PublishGitHubRelease + displayName: Publish GitHub Release + dependsOn: BlobPublic + jobs: + - template: /.pipelines/templates/release-githubtasks.yml@self + + - stage: PublishNuGet + displayName: Publish NuGet + dependsOn: PublishGitHubRelease + jobs: + - template: /.pipelines/templates/release-publish-nuget.yml@self + parameters: + skipPublish: true + + - stage: PublishPMC + displayName: Publish PMC + dependsOn: PublishGitHubRelease + jobs: + - template: /.pipelines/templates/release-publish-pmc.yml@self + parameters: + skipPublish: true + + - stage: ReleaseDocker + dependsOn: PublishGitHubRelease + displayName: 'Docker Release' + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start Docker Build + jobName: StartDockerBuild + instructions: | + Kick off Docker build + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start Docker Release + dependsOnJob: StartDockerBuild + jobName: StartDockerRelease + instructions: | + Kickoff docker release + + - stage: UpdateDotnetDocker + dependsOn: PublishGitHubRelease + displayName: Update DotNet SDK Docker images + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Update .NET SDK docker images + jobName: DotnetDocker + instructions: | + Create PR for updating dotnet-docker images to use latest PowerShell version. + 1. Fork and clone https://github.com/dotnet/dotnet-docker.git + 2. git checkout upstream/nightly -b updatePS + 3. dotnet run --project .\eng\update-dependencies\ -- --product-version powershell= --compute-shas + 4. create PR targeting nightly branch + + - stage: UpdateWinGet + dependsOn: PublishGitHubRelease + displayName: Add manifest entry to winget + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Add manifest entry to winget + jobName: UpdateWinGet + instructions: | + This is typically done by the community 1-2 days after the release. + + - stage: PublishMsix + dependsOn: PublishGitHubRelease + displayName: Publish MSIX to store + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Publish the MSIX Bundle package to store + jobName: PublishMsix + instructions: | + Ask Steve to release MSIX bundle package to Store + + - stage: PublishVPack + dependsOn: PublishGitHubRelease + displayName: Release vPack + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start vPack Release pipeline + jobName: PublishVPack + instructions: | + Kick off vPack release pipeline + + # Need to verify if the Az PS / CLI team still uses this. Skippinng for this release. + # - stage: ReleaseDeps + # dependsOn: GitHubTasks + # displayName: Update pwsh.deps.json links + # jobs: + # - template: templates/release-UpdateDepsJson.yml + + - stage: UploadBuildInfoJson + dependsOn: PublishGitHubRelease + displayName: Upload BuildInfo.json + jobs: + - template: /.pipelines/templates/release-upload-buildinfo.yml@self + + - stage: ReleaseSymbols + dependsOn: PublishGitHubRelease + displayName: Release Symbols + jobs: + - template: /.pipelines/templates/release-symbols.yml@self + + - stage: ChangesToMaster + displayName: Ensure changes are in GH master + dependsOn: ['PublishNuGet', 'PublishPMC'] + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make sure changes are in master + jobName: MergeToMaster + instructions: | + Make sure that changes README.md and metadata.json are merged into master on GitHub. + + - stage: ReleaseSnap + displayName: Release Snap + dependsOn: 'ChangesToMaster' + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Publish Snap + jobName: PublishSnapJob + instructions: | + Publish Snap + + - stage: ReleaseToMU + displayName: Release to MU + dependsOn: ['PublishNuGet', 'PublishPMC', 'ChangesToMaster'] + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Release to MU + instructions: | + Notify the PM team to start the process of releasing to MU. + + - stage: ReleaseClose + displayName: Finish Release + dependsOn: ['ReleaseToMU', 'ReleaseSymbols', 'ReleaseSnap'] + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Retain Build + jobName: RetainBuild + instructions: | + Retain the build + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Delete release branch + jobName: DeleteBranch + instructions: | + Delete release diff --git a/.pipelines/templates/approvalJob.yml b/.pipelines/templates/approvalJob.yml new file mode 100644 index 0000000000..ac3b8bc2ab --- /dev/null +++ b/.pipelines/templates/approvalJob.yml @@ -0,0 +1,36 @@ +parameters: + - name: displayName + type: string + - name: instructions + type: string + - name: jobName + type: string + default: approval + - name: timeoutInMinutes + type: number + # 2 days + default: 2880 + - name: onTimeout + type: string + default: 'reject' + values: + - resume + - reject + - name: dependsOnJob + type: string + default: '' + +jobs: + - job: ${{ parameters.jobName }} + dependsOn: ${{ parameters.dependsOnJob }} + displayName: ${{ parameters.displayName }} + pool: + type: agentless + timeoutInMinutes: 4320 # job times out in 3 days + steps: + - task: ManualValidation@0 + displayName: ${{ parameters.displayName }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + inputs: + instructions: ${{ parameters.instructions }} + onTimeout: ${{ parameters.onTimeout }} diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml new file mode 100644 index 0000000000..3b10b26958 --- /dev/null +++ b/.pipelines/templates/release-MakeBlobPublic.yml @@ -0,0 +1,170 @@ +jobs: +- template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Approve Blob Public + jobName: ApproveBlobPublic + instructions: | + Are you sure you want to make the blob public? + +- job: blobPublic + displayName: Make Azure Blob Public + dependsOn: ApproveBlobPublic + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'Azure Blob variable group' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + displayName: Remove AzRM modules + + - task: AzurePowerShell@5 + displayName: Set blob permissions + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $storageAccount = "$(StorageAccount)" + $containerName = "$(azureVersion)" + $publicAccess = "blob" + + Set-AzStorageContainerAcl ` + -Context (New-AzStorageContext -StorageAccountName $storageAccount) ` + -Container $containerName ` + -PublicAccess $publicAccess + + Set-AzStorageContainerAcl ` + -Context (New-AzStorageContext -StorageAccountName $storageAccount) ` + -Container "$containerName-gc" ` + -PublicAccess $publicAccess + +- template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Approve Copy Global tool packages to PSInfra storage + jobName: CopyBlobApproval + dependsOnJob: blobPublic + instructions: | + Approval for Copy global tool packages to PSInfra storage + +- job: PSInfraBlobPublic + displayName: Copy global tools to PSInfra storage + dependsOn: CopyBlobApproval + pool: + type: windows + + variables: + - group: 'PSInfraStorage' + - group: 'Azure Blob variable group' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + displayName: Remove AzRM modules + + - task: AzurePowerShell@5 + displayName: Copy blobs to PSInfra storage + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $sourceStorageAccountName = '$(StorageAccount)' + $sourceContainerName = '$(AzureVersion)' + + $destinationStorageAccountName = '$(PSInfraStorageAccount)' + $destinationContainerName = "tool/$(Version)" + + $sourceContext = New-AzStorageContext -StorageAccountName $sourceStorageAccountName + $destinationContext = New-AzStorageContext -StorageAccountName $destinationStorageAccountName + + $blobs = Get-AzStorageBlob -Context $sourceContext -Container $sourceContainerName + + foreach ($blob in $blobs) { + $sourceBlobName = $blob.Name + $destinationBlobName = $sourceBlobName + + $destinationBlob = Start-AzStorageBlobCopy -Context $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName + + # Wait for the copy operation to complete + $destinationBlob | Wait-AzStorageBlobCopy + + if ($destinationBlob.CopyStatus -eq "Success") { + Write-Host "Blob copy completed successfully for $sourceBlobName." + } else { + Write-Host "Blob copy failed for $sourceBlobName." + } + } diff --git a/.pipelines/templates/release-SetReleaseTagandContainerName.yml b/.pipelines/templates/release-SetReleaseTagandContainerName.yml new file mode 100644 index 0000000000..7e88624b45 --- /dev/null +++ b/.pipelines/templates/release-SetReleaseTagandContainerName.yml @@ -0,0 +1,26 @@ +steps: +- pwsh: | + $variable = 'releaseTag' + $branch = $ENV:BUILD_SOURCEBRANCH + if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') + { + throw "Branch name is not in release format: '$branch'" + } + + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' + $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" + Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set Release Tag + +- pwsh: | + $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set container name diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml new file mode 100644 index 0000000000..da5e5136a5 --- /dev/null +++ b/.pipelines/templates/release-create-msix.yml @@ -0,0 +1,100 @@ +jobs: +- job: CreateMSIXBundle + displayName: Create .msixbundle file + pool: + type: windows + + variables: + - group: msixTools + - group: 'Azure Blob variable group' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + + steps: + - template: release-SetReleaseTagandContainerName.yml@self + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_arm64 + displayName: Download arm64 msix + patterns: '**/*.msix' + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_x64 + displayName: Download x64 msix + patterns: '**/*.msix' + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_x86 + displayName: Download x86 msix + patterns: '**/*.msix' + + - pwsh: | + $cmd = Get-Command makeappx.exe -ErrorAction Ignore + if ($cmd) { + Write-Verbose -Verbose 'makeappx available in PATH' + $exePath = $cmd.Source + } else { + $toolsDir = '$(Pipeline.Workspace)\releasePipeline\tools' + New-Item $toolsDir -Type Directory -Force > $null + Invoke-RestMethod -Uri '$(makeappUrl)' -OutFile "$toolsDir\makeappx.zip" + Expand-Archive "$toolsDir\makeappx.zip" -DestinationPath "$toolsDir\makeappx" -Force + $exePath = "$toolsDir\makeappx\makeappx.exe" + + Write-Verbose -Verbose 'makeappx was installed:' + Get-ChildItem -Path $toolsDir -Recurse + } + + $vstsCommandString = "vso[task.setvariable variable=MakeAppxPath]$exePath" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install makeappx tool + retryCountOnTaskFailure: 1 + + - pwsh: | + $sourceDir = '$(Pipeline.Workspace)\releasePipeline\msix' + $null = New-Item -Path $sourceDir -ItemType Directory -Force + + $msixFiles = Get-ChildItem -Path "$(Pipeline.Workspace)/PSPackagesOfficial/*.msix" -Recurse + foreach ($msixFile in $msixFiles) { + $null = Copy-Item -Path $msixFile.FullName -Destination $sourceDir -Force -Verbose + } + + $file = Get-ChildItem $sourceDir | Select-Object -First 1 + $prefix = ($file.BaseName -split "-win")[0] + $pkgName = "$prefix.msixbundle" + Write-Verbose -Verbose "Creating $pkgName" + + $makeappx = '$(MakeAppxPath)' + $outputDir = "$sourceDir\output" + New-Item $outputDir -Type Directory -Force > $null + & $makeappx bundle /d $sourceDir /p "$outputDir\$pkgName" + + Get-ChildItem -Path $sourceDir -Recurse + $vstsCommandString = "vso[task.setvariable variable=BundleDir]$outputDir" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Create MsixBundle + retryCountOnTaskFailure: 1 + + - task: AzurePowerShell@5 + displayName: Upload msix to blob + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $containerName = '$(AzureVersion)-private' + $storageAccount = '$(StorageAccount)' + + $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount + + if ($env:BundleDir) { + $bundleFile = "$env:BundleDir\*.msixbundle" + $blobName = Get-Item $bundleFile | Split-Path -Leaf + Write-Verbose -Verbose "Uploading $bundleFile to $containerName/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + } + else{ + throw "BundleDir not found" + } diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml new file mode 100644 index 0000000000..e94fa16fef --- /dev/null +++ b/.pipelines/templates/release-githubtasks.yml @@ -0,0 +1,160 @@ +jobs: +- job: GithubReleaseDraft + displayName: Create GitHub Release Draft + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - group: 'Azure Blob variable group' + - group: 'GitHubTokens' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - pwsh: | + # Uninstall Azure RM modules + $azRmModules = Get-Module -Name AzureRM* -ListAvailable + if ($azRmModules) { + $azRmModules | Remove-Module -Force + } + + # Install Az.Storage module if not already installed + if (-not (Get-Module -Name Az.Storage -ListAvailable)) { + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + } + displayName: Install Az.Storage module + + - task: AzurePowerShell@5 + displayName: Download packages from Azure Storage + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $storageAccount = "$(StorageAccount)" + $containerName = "$(AzureVersion)" + $destinationPath = "$(System.ArtifactsDirectory)" + + # Get storage account context + $storageContext = New-AzStorageContext -StorageAccountName $storageAccount + + $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext + foreach ($blob in $blobList) { + $blobName = $blob.Name + $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName + Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force + Write-Output "Downloaded $blobName to $destinationFile" + } + + - pwsh: | + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty FullName + displayName: Capture downloaded artifacts + + - pwsh: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + displayName: Clone Internal-Tools repository + + - pwsh: | + $Path = "$(System.ArtifactsDirectory)" + $OutputPath = Join-Path $Path 'hashes.sha256' + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + displayName: Add sha256 hashes + + - pwsh: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Set release version' + + - pwsh: | + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + $isPreview = $semanticVersion.PreReleaseLabel -ne $null + + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" + $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + + $clContent = $changelog | ForEach-Object { + if ($_ -match $startPattern) { $outputLine = $true } + elseif ($_ -match $endPattern) { $outputLine = $false } + if ($outputLine) { $_} + } | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) + displayName: Publish Release Draft + +- template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + dependsOnJob: GithubReleaseDraft + instructions: | + Push the git tag to upstream + +- template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + jobName: DraftPublic + dependsOnJob: PushGitTag + instructions: | + Make the GitHub Release Draft Public diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml new file mode 100644 index 0000000000..c94efc52e1 --- /dev/null +++ b/.pipelines/templates/release-publish-nuget.yml @@ -0,0 +1,71 @@ +parameters: + - name: skipPublish + default: false + type: boolean + +jobs: +- job: NuGetPublish + displayName: Publish to NuGet + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - download: PSPackagesOfficial + artifact: drop_nupkg_build_nupkg + displayName: Download nuget packages + + - pwsh: | + #Exclude all global tool packages. Their names start with 'PowerShell.' + $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" + Copy-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose + + $releaseVersion = '$(VERSION)' + $globalToolPath = "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/PowerShell.$releaseVersion.nupkg" + + if ($releaseVersion -notlike '*-*') { + # Copy the global tool package for stable releases + Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" + } + + Get-ChildItem "$(Pipeline.Workspace)/release" -recurse + displayName: Download and capture nupkgs + condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) + + - task: NuGetCommand@2 + displayName: 'NuGet push' + condition: and(eq('${{ parameters.skipPublish }}', 'false'), succeeded()) + inputs: + command: push + packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' + nuGetFeedType: external + publishFeedCredentials: PowerShellNuGetOrgPush diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml new file mode 100644 index 0000000000..0d752b09e1 --- /dev/null +++ b/.pipelines/templates/release-publish-pmc.yml @@ -0,0 +1,83 @@ +parameters: + - name: skipPublish + default: false + type: boolean + +jobs: +- job: PMCPublish + displayName: Publish to PMC + condition: succeeded() + pool: + type: linux + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set Package version + + - pwsh: | + $branch = 'main-mirror' + $gitArgs = "clone", + "--verbose", + "--branch", + "$branch", + "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", + '$(Pipeline.Workspace)/tools' + $gitArgs | Write-Verbose -Verbose + git $gitArgs + displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'pmc' + pythonDownloadServiceConnections: pmcDownload + + - pwsh: | + pip install pmc-cli + + $newPath = (resolve-path '~/.local/bin').providerpath + $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install pmc cli + + - pwsh: | + $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json + $params = @{ + ReleaseTag = "$(ReleaseTag)" + AadClientId = "$(PmcCliClientID)" + BlobFolderName = "$(AzureVersion)" + LTS = $metadata.LTSRelease.Latest + ForProduction = $true + SkipPublish = $${{ parameters.skipPublish }} + MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' + } + + $params | Out-String -width 9999 -Stream | write-Verbose -Verbose + + & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params + displayName: Run release script diff --git a/.pipelines/templates/release-symbols.yml b/.pipelines/templates/release-symbols.yml new file mode 100644 index 0000000000..ec8e91d2fb --- /dev/null +++ b/.pipelines/templates/release-symbols.yml @@ -0,0 +1,85 @@ +parameters: + - name: skipPublish + default: false + type: boolean + +jobs: +- job: PublishSymbols + displayName: Publish Symbols + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_x64_release + patterns: 'symbols.zip' + displayName: Download winx64 + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_x86_release + patterns: 'symbols.zip' + displayName: Download winx86 + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_arm64_release + patterns: 'symbols.zip' + displayName: Download winx64 + + - pwsh: | + Write-Verbose -Verbose "Enumerating $(Pipeline.Workspace)\CoOrdinatedBuildPipeline" + $downloadedArtifacts = Get-ChildItem -Recurse "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline" -Recurse -Filter 'symbols.zip' + $downloadedArtifacts + $expandedRoot = New-Item -Path "$(Pipeline.Workspace)/expanded" -ItemType Directory -Verbose + $symbolsRoot = New-Item -Path "$(Pipeline.Workspace)/symbols" -ItemType Directory -Verbose + + $downloadedArtifacts | ForEach-Object { + $folderName = (Get-Item (Split-Path $_.FullName)).Name + $destFolder = New-Item -Path "$expandedRoot/$folderName/$($_.BaseName)/" -ItemType Directory -Verbose + Expand-Archive -Path $_.FullName -DestinationPath $destFolder -Force + + $symbolsToPublish = New-Item -Path "$symbolsRoot/$($_.BaseName)" -ItemType Directory -Verbose + + Get-ChildItem -Path $destFolder -Recurse -Filter '*.pdb' | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $symbolsToPublish -Verbose + } + } + + Write-Verbose -Verbose "Enumerating $symbolsRoot" + Get-ChildItem -Path $symbolsRoot -Recurse + $vstsCommandString = "vso[task.setvariable variable=SymbolsPath]$symbolsRoot" + Write-Verbose -Message "$vstsCommandString" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Expand and capture symbols folders + + - task: PublishSymbols@2 + inputs: + symbolsFolder: '$(SymbolsPath)' + searchPattern: '**/*.pdb' + indexSources: false + publishSymbols: true + symbolServerType: teamServices + detailedLog: true diff --git a/.pipelines/templates/release-upload-buildinfo.yml b/.pipelines/templates/release-upload-buildinfo.yml new file mode 100644 index 0000000000..b13c1ee73f --- /dev/null +++ b/.pipelines/templates/release-upload-buildinfo.yml @@ -0,0 +1,129 @@ +parameters: + - name: skipPublish + default: false + type: boolean + +jobs: +- job: PMCPublish + displayName: Publish to PMC + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - download: PSPackagesOfficial + artifact: BuildInfoJson + displayName: Download build info artifact + + - pwsh: | + Import-Module '$(Build.SourcesDirectory)/tools/ci.psm1' + $jsonFile = Get-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/BuildInfoJson/*.json" + $fileName = Split-Path $jsonFile -Leaf + + $dateTime = [datetime]::UtcNow + $dateTime = [datetime]::new($dateTime.Ticks - ($dateTime.Ticks % [timespan]::TicksPerSecond), $dateTime.Kind) + + $metadata = Get-Content ./tools/metadata.json | ConvertFrom-Json + $stableRelease = $metadata.StableRelease.Latest + $ltsRelease = $metadata.LTSRelease.Latest + + Write-Verbose -Verbose "Writing $jsonFile contents:" + $buildInfoJsonContent = Get-Content $jsonFile -Encoding UTF8NoBom -Raw + Write-Verbose -Verbose $buildInfoJsonContent + + $buildInfo = $buildInfoJsonContent | ConvertFrom-Json + $buildInfo.ReleaseDate = $dateTime + + $targetFile = "$ENV:PIPELINE_WORKSPACE/$fileName" + ConvertTo-Json -InputObject $buildInfo | Out-File $targetFile -Encoding ascii + + if ($stableRelease -or $fileName -eq "preview.json") { + Set-BuildVariable -Name CopyMainBuildInfo -Value YES + } else { + Set-BuildVariable -Name CopyMainBuildInfo -Value NO + } + + Set-BuildVariable -Name BuildInfoJsonFile -Value $targetFile + + ## Create 'lts.json' if it's the latest stable and also a LTS release. + + if ($fileName -eq "stable.json") { + if ($ltsRelease) { + $ltsFile = "$ENV:PIPELINE_WORKSPACE/lts.json" + Copy-Item -Path $targetFile -Destination $ltsFile -Force + Set-BuildVariable -Name LtsBuildInfoJsonFile -Value $ltsFile + Set-BuildVariable -Name CopyLTSBuildInfo -Value YES + } else { + Set-BuildVariable -Name CopyLTSBuildInfo -Value NO + } + + $releaseTag = $buildInfo.ReleaseTag + $version = $releaseTag -replace '^v' + $semVersion = [System.Management.Automation.SemanticVersion] $version + + $versionFile = "$ENV:PIPELINE_WORKSPACE/$($semVersion.Major)-$($semVersion.Minor).json" + Copy-Item -Path $targetFile -Destination $versionFile -Force + Set-BuildVariable -Name VersionBuildInfoJsonFile -Value $versionFile + Set-BuildVariable -Name CopyVersionBuildInfo -Value YES + } else { + Set-BuildVariable -Name CopyVersionBuildInfo -Value NO + } + displayName: Create json files + + - task: AzurePowerShell@5 + displayName: Upload buildjson to blob + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $containerName = "BuildInfo" + $storageAccount = '$(StorageAccount)' + + $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount + + if ($env:CopyMainBuildInfo -eq 'YES') { + $jsonFile = "$env:BuildInfoJsonFile" + $blobName = Get-Item $jsonFile | Split-Path -Leaf + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + } + + if ($env:CopyLTSBuildInfo -eq 'YES') { + $jsonFile = "$env:LtsBuildInfoJsonFile" + $blobName = Get-Item $jsonFile | Split-Path -Leaf + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + } + + if ($env:CopyVersionBuildInfo -eq 'YES') { + $jsonFile = "$env:VersionBuildInfoJsonFile" + $blobName = Get-Item $jsonFile | Split-Path -Leaf + Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName" + Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext + } + condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES')) diff --git a/.pipelines/templates/release-validate-fxdpackages.yml b/.pipelines/templates/release-validate-fxdpackages.yml new file mode 100644 index 0000000000..62e907fcf3 --- /dev/null +++ b/.pipelines/templates/release-validate-fxdpackages.yml @@ -0,0 +1,132 @@ +parameters: + jobName: "" + displayName: "" + jobtype: "" + artifactName: "" + packageNamePattern: "" + arm64: "no" + +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + variables: + - group: DotNetPrivateBuildAccess + - name: artifactName + value: ${{ parameters.artifactName }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + + pool: + type: ${{ parameters.jobtype }} + ${{ if eq(parameters.arm64, 'yes') }}: + hostArchitecture: arm64 + + steps: + - checkout: self + clean: true + + - template: release-SetReleaseTagandContainerName.yml@self + + - download: PSPackagesOfficial + artifact: "${{ parameters.artifactName }}" + displayName: Download fxd artifact + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + + - pwsh: | + $artifactName = '$(artifactName)' + Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/$artifactName" -Recurse + displayName: 'Capture Downloaded Artifacts' + + - pwsh: | + $repoRoot = "$(Build.SourcesDirectory)/PowerShell" + $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" + $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json + + # Channel is like: $Channel = "5.0.1xx-preview2" + $Channel = $dotnetMetadataJson.sdk.channel + + $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version + Import-Module "$repoRoot/build.psm1" -Force + + Find-Dotnet + + if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) + { + $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value + if ($nugetFeed) { + Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet + Write-Verbose -Message "Register new package source 'dotnet'" -verbose + } + } + + ## Install latest version from the channel + + #Install-Dotnet -Channel "$Channel" -Version $sdkVersion + Start-PSBootstrap + + Write-Verbose -Message "Installing .NET SDK completed." -Verbose + + displayName: Install .NET + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $artifactName = '$(artifactName)' + $rootPath = "$(Pipeline.Workspace)/PSPackagesOfficial/$artifactName" + + $destPath = New-Item "$rootPath/fxd" -ItemType Directory + $packageNameFilter = '${{ parameters.packageNamePattern }}' + + if ($packageNameFilter.EndsWith('tar.gz')) { + $package = @(Get-ChildItem -Path "$rootPath/*.tar.gz") + Write-Verbose -Verbose "Package: $package" + if ($package.Count -ne 1) { + throw 'Only 1 package was expected.' + } + tar -xvf $package.FullName -C $destPath + } + else { + $package = @(Get-ChildItem -Path "$rootPath/*.zip") + Write-Verbose -Verbose "Package: $package" + if ($package.Count -ne 1) { + throw 'Only 1 package was expected.' + } + Expand-Archive -Path $package.FullName -Destination "$destPath" -Verbose + } + displayName: Expand fxd package + + - pwsh: | + $repoRoot = "$(Build.SourcesDirectory)/PowerShell" + $artifactName = '$(artifactName)' + $rootPath = "$(Pipeline.Workspace)/PSPackagesOfficial/$artifactName" + + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + Import-Module "$repoRoot/build.psm1" -Force + Find-Dotnet -SetDotnetRoot + Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" + Write-Verbose -Verbose "Check dotnet install" + dotnet --info + Write-Verbose -Verbose "Start test" + $packageNameFilter = '${{ parameters.packageNamePattern }}' + $pwshExeName = if ($packageNameFilter.EndsWith('tar.gz')) { 'pwsh' } else { 'pwsh.exe' } + $pwshPath = Join-Path "$rootPath/fxd" $pwshExeName + + if ($IsLinux) { + chmod u+x $pwshPath + } + + $pwshDllPath = Join-Path "$rootPath/fxd" 'pwsh.dll' + + $actualOutput = & dotnet $pwshDllPath -c 'Start-ThreadJob -ScriptBlock { "1" } | Wait-Job | Receive-Job' + Write-Verbose -Verbose "Actual output: $actualOutput" + if ($actualOutput -ne 1) { + throw "Actual output is not as expected" + } + displayName: Test package diff --git a/.pipelines/templates/release-validate-globaltools.yml b/.pipelines/templates/release-validate-globaltools.yml new file mode 100644 index 0000000000..fba8b7b3f9 --- /dev/null +++ b/.pipelines/templates/release-validate-globaltools.yml @@ -0,0 +1,157 @@ +parameters: + jobName: "" + displayName: "" + jobtype: "windows" + globalToolExeName: 'pwsh.exe' + globalToolPackageName: 'PowerShell.Windows.x64' + + +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + pool: + type: ${{ parameters.jobtype }} + variables: + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + + steps: + - checkout: self + clean: true + + - template: release-SetReleaseTagandContainerName.yml@self + + - download: PSPackagesOfficial + artifact: drop_nupkg_build_nupkg + displayName: Download nupkgs + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + + - pwsh: | + Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" -Recurse + displayName: 'Capture Downloaded Artifacts' + + - pwsh: | + $repoRoot = "$(Build.SourcesDirectory)/PowerShell" + $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" + $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json + + # Channel is like: $Channel = "5.0.1xx-preview2" + $Channel = $dotnetMetadataJson.sdk.channel + + $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version + Import-Module "$repoRoot/build.psm1" -Force + + Find-Dotnet + + if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) + { + $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value + if ($nugetFeed) { + Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet + Write-Verbose -Message "Register new package source 'dotnet'" -verbose + } + } + + ## Install latest version from the channel + + #Install-Dotnet -Channel "$Channel" -Version $sdkVersion + Start-PSBootstrap + + Write-Verbose -Message "Installing .NET SDK completed." -Verbose + + displayName: Install .NET + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $repoRoot = "$(Build.SourcesDirectory)/PowerShell" + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + Import-Module "$repoRoot/build.psm1" -Force + Start-PSBootstrap + + $toolPath = New-Item -ItemType Directory "$(System.DefaultWorkingDirectory)/toolPath" | Select-Object -ExpandProperty FullName + + Write-Verbose -Verbose "dotnet tool list -g" + dotnet tool list -g + + $packageName = '${{ parameters.globalToolPackageName }}' + Write-Verbose -Verbose "Installing $packageName" + + dotnet tool install --add-source "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg" --tool-path $toolPath --version '$(Version)' $packageName + + Get-ChildItem -Path $toolPath + + displayName: Install global tool + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" + + if (-not (Test-Path $toolPath)) + { + throw "Tool is not installed at $toolPath" + } + else + { + Write-Verbose -Verbose "Tool found at: $toolPath" + } + displayName: Validate tool is installed + + - pwsh: | + $repoRoot = "$(Build.SourcesDirectory)/PowerShell" + Import-Module "$repoRoot/build.psm1" -Force + Start-PSBootstrap + + $exeName = if ($IsWindows) { "pwsh.exe" } else { "pwsh" } + + $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" + + $source = (get-command -Type Application -Name dotnet | Select-Object -First 1 -ExpandProperty source) + $target = (Get-ChildItem $source).target + + # If we find a symbolic link for dotnet, then we need to split the filename off the target. + if ($target) { + Write-Verbose -Verbose "Splitting target: $target" + $target = Split-Path $target + } + + Write-Verbose -Verbose "target is set as $target" + + $env:DOTNET_ROOT = (resolve-path -Path (Join-Path (split-path $source) $target)).ProviderPath + + Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" + Get-ChildItem $env:DOTNET_ROOT + + $versionFound = & $toolPath -c '$PSVersionTable.PSVersion.ToString()' + + if ( '$(Version)' -ne $versionFound) + { + throw "Expected version of global tool not found. Installed version is $versionFound" + } + else + { + write-verbose -verbose "Found expected version: $versionFound" + } + + $dateYear = & $toolPath -c '(Get-Date).Year' + + if ( $dateYear -ne [DateTime]::Now.Year) + { + throw "Get-Date returned incorrect year: $dateYear" + } + else + { + write-verbose -verbose "Got expected year: $dateYear" + } + displayName: Basic validation + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) diff --git a/.pipelines/templates/release-validate-packagenames.yml b/.pipelines/templates/release-validate-packagenames.yml new file mode 100644 index 0000000000..564b9d7c78 --- /dev/null +++ b/.pipelines/templates/release-validate-packagenames.yml @@ -0,0 +1,193 @@ +jobs: +- job: validatePackageNames + displayName: Validate Package Names + pool: + type: windows + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - group: 'Azure Blob variable group' + + steps: + - pwsh: | + Get-ChildItem ENV: + displayName: Capture environment + + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks + Write-Host $name + Write-Host "##vso[build.updatebuildnumber]$name" + displayName: Set Release Name + + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + + displayName: Remove AzRM modules and install Az.Storage + + - task: AzurePowerShell@5 + displayName: Upload packages to blob + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $storageAccount = Get-AzStorageAccount -ResourceGroupName '$(StorageResourceGroup)' -Name '$(StorageAccount)' + $ctx = $storageAccount.Context + $container = '$(AzureVersion)' + + $destinationPath = '$(System.ArtifactsDirectory)' + $blobList = Get-AzStorageBlob -Container $container -Context $ctx + foreach ($blob in $blobList) { + $blobName = $blob.Name + $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName + Get-AzStorageBlobContent -Container $container -Blob $blobName -Destination $destinationFile -Context $ctx -Force + Write-Output "Downloaded $blobName to $destinationFile" + } + + - pwsh: | + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name + displayName: Capture Artifact Listing + + - pwsh: | + $message = @() + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.rpm | ForEach-Object { + if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d+\.\d+\.\d+(_[a-z]*\.\d+)?-1.(rh|cm).(x86_64|aarch64)\.rpm') + { + $messageInstance = "$($_.Name) is not a valid package name" + $message += $messageInstance + Write-Warning $messageInstance + } + } + if($message.count -gt 0){throw ($message | out-string)} + displayName: Validate RPM package names + + - pwsh: | + $message = @() + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.tar.gz | ForEach-Object { + if($_.Name -notmatch 'powershell-(lts-)?\d+\.\d+\.\d+\-([a-z]*.\d+\-)?(linux|osx|linux-musl)+\-(x64\-fxdependent|x64|arm32|arm64|x64\-musl-noopt\-fxdependent)\.(tar\.gz)') + { + $messageInstance = "$($_.Name) is not a valid package name" + $message += $messageInstance + Write-Warning $messageInstance + } + } + if($message.count -gt 0){throw ($message | out-string)} + displayName: Validate Tar.Gz Package Names + + - pwsh: | + $message = @() + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.pkg | ForEach-Object { + if($_.Name -notmatch 'powershell-(lts-)?\d+\.\d+\.\d+\-([a-z]*.\d+\-)?osx(\.10\.12)?\-(x64|arm64)\.pkg') + { + $messageInstance = "$($_.Name) is not a valid package name" + $message += $messageInstance + Write-Warning $messageInstance + } + } + if($message.count -gt 0){throw ($message | out-string)} + displayName: Validate PKG Package Names + + - pwsh: | + $message = @() + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -include *.zip, *.msi | ForEach-Object { + if($_.Name -notmatch 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(fxdependent|x64|arm64|x86|fxdependentWinDesktop)\.(msi|zip){1}') + { + $messageInstance = "$($_.Name) is not a valid package name" + $message += $messageInstance + Write-Warning $messageInstance + } + } + + if($message.count -gt 0){throw ($message | out-string)} + displayName: Validate Zip and MSI Package Names + + - pwsh: | + $message = @() + Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.deb | ForEach-Object { + if($_.Name -notmatch 'powershell(-preview|-lts)?_\d+\.\d+\.\d+([\-~][a-z]*.\d+)?-\d\.deb_amd64\.deb') + { + $messageInstance = "$($_.Name) is not a valid package name" + $message += $messageInstance + Write-Warning $messageInstance + } + } + if($message.count -gt 0){throw ($message | out-string)} + displayName: Validate Deb Package Names + +# Move to 1ES SBOM validation tool +# - job: validateBOM +# displayName: Validate Package Names +# pool: +# type: windows +# variables: +# - name: ob_outputDirectory +# value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' +# - name: ob_sdl_credscan_suppressionsFile +# value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json +# - name: ob_sdl_tsa_configFile +# value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json +# - group: 'Azure Blob variable group' + +# steps: +# - checkout: self +# clean: true + +# - pwsh: | +# Get-ChildItem ENV: +# displayName: Capture environment + +# - template: release-SetReleaseTagAndContainerName.yml + +# - pwsh: | +# $name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks +# Write-Host $name +# Write-Host "##vso[build.updatebuildnumber]$name" +# displayName: Set Release Name + +# - task: DownloadPipelineArtifact@2 +# inputs: +# source: specific +# project: PowerShellCore +# pipeline: '696' +# preferTriggeringPipeline: true +# runVersion: latestFromBranch +# runBranch: '$(Build.SourceBranch)' +# artifact: finalResults +# path: $(System.ArtifactsDirectory) + + +# - pwsh: | +# Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name +# displayName: Capture Artifact Listing + +# - pwsh: | +# Install-module Pester -Scope CurrentUser -Force -MaximumVersion 4.99 +# displayName: Install Pester +# condition: succeededOrFailed() + +# - pwsh: | +# Import-module './build.psm1' +# Import-module './tools/packaging' +# $env:PACKAGE_FOLDER = '$(System.ArtifactsDirectory)' +# $path = Join-Path -Path $pwd -ChildPath './packageReleaseTests.xml' +# $results = invoke-pester -Script './tools/packaging/releaseTests' -OutputFile $path -OutputFormat NUnitXml -PassThru +# Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Package Release Tests;publishRunAttachments=true;resultFiles=$path;]" +# if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0) +# { +# throw "Package Release Tests failed" +# } +# displayName: Run packaging release tests diff --git a/.pipelines/templates/release-validate-sdk.yml b/.pipelines/templates/release-validate-sdk.yml new file mode 100644 index 0000000000..8a73fe6f1e --- /dev/null +++ b/.pipelines/templates/release-validate-sdk.yml @@ -0,0 +1,127 @@ +parameters: + jobName: "" + displayName: "" + jobtype: "windows" + +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + pool: + ${{ if eq(parameters.jobtype, 'macos') }}: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-latest' + ${{ else }}: + type: ${{ parameters.jobtype }} + + variables: + - group: AzDevOpsArtifacts + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + + steps: + - checkout: self + clean: true + + - template: release-SetReleaseTagandContainerName.yml@self + + - download: PSPackagesOfficial + artifact: drop_nupkg_build_nupkg + displayName: Download nupkgs + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + + - pwsh: | + Get-ChildItem "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" -Recurse + displayName: 'Capture Downloaded Artifacts' + + - pwsh: | + $repoRoot = $isMacOS ? "$(Build.SourcesDirectory)" : "$(Build.SourcesDirectory)/PowerShell" + + $dotnetMetadataPath = "$repoRoot/DotnetRuntimeMetadata.json" + $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json + + # Channel is like: $Channel = "5.0.1xx-preview2" + $Channel = $dotnetMetadataJson.sdk.channel + + $sdkVersion = (Get-Content "$repoRoot/global.json" -Raw | ConvertFrom-Json).sdk.version + Import-Module "$repoRoot/build.psm1" -Force + + Find-Dotnet + + if(-not (Get-PackageSource -Name 'dotnet' -ErrorAction SilentlyContinue)) + { + $nugetFeed = ([xml](Get-Content $repoRoot/nuget.config -Raw)).Configuration.packagesources.add | Where-Object { $_.Key -eq 'dotnet' } | Select-Object -ExpandProperty Value + + if ($nugetFeed) { + Register-PackageSource -Name 'dotnet' -Location $nugetFeed -ProviderName NuGet + Write-Verbose -Message "Register new package source 'dotnet'" -verbose + } + } + + ## Install latest version from the channel + #Install-Dotnet -Channel "$Channel" -Version $sdkVersion + + Start-PSBootstrap + + Write-Verbose -Message "Installing .NET SDK completed." -Verbose + + displayName: Install .NET + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $repoRoot = $isMacOS ? "$(Build.SourcesDirectory)" : "$(Build.SourcesDirectory)/PowerShell" + + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + Import-Module "$repoRoot/build.psm1" -Force + Start-PSBootstrap + + $localLocation = "$(Pipeline.Workspace)/PSPackagesOfficial/drop_nupkg_build_nupkg" + $xmlElement = @" + + + + "@ + + $releaseVersion = '$(Version)' + + Write-Verbose -Message "Release Version: $releaseVersion" -Verbose + + Set-Location -Path $repoRoot/test/hosting + + Get-ChildItem + + ## register the packages download directory in the nuget file + $nugetConfigContent = Get-Content ./NuGet.Config -Raw + $updateNugetContent = $nugetConfigContent.Replace("", $xmlElement) + + $updateNugetContent | Out-File ./NuGet.Config -Encoding ascii + + Get-Content ./NuGet.Config + + # Add workaround to unblock xUnit testing see issue: https://github.com/dotnet/sdk/issues/26462 + $dotnetPath = if ($IsWindows) { "$env:LocalAppData\Microsoft\dotnet" } else { "$env:HOME/.dotnet" } + $env:DOTNET_ROOT = $dotnetPath + + dotnet --info + dotnet restore + dotnet test /property:RELEASE_VERSION=$releaseVersion --test-adapter-path:. "--logger:xunit;LogFilePath=$(System.DefaultWorkingDirectory)/test-hosting.xml" + + displayName: Restore and execute tests + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - task: PublishTestResults@2 + displayName: 'Publish Test Results **\test-hosting.xml' + inputs: + testResultsFormat: XUnit + testResultsFiles: '**\test-hosting.xml' diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index d6be94368b..e43ba5c0d8 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -106,6 +106,8 @@ jobs: $pdbs | ForEach-Object { Write-Verbose -Verbose "Pdb: $($_.FullName)" } + + $pdbs | Compress-Archive -DestinationPath "$(ob_outputDirectory)/symbols.zip" -Update } if ($runtime -eq 'fxdependent') diff --git a/build.psm1 b/build.psm1 index 3b7adca6f3..9362965888 100644 --- a/build.psm1 +++ b/build.psm1 @@ -2297,7 +2297,7 @@ function Start-PSBootstrap { } if ($Package) { Import-Module "$PSScriptRoot\tools\wix\wix.psm1" - $isArm64 = '$(Runtime)' -eq 'arm64' + $isArm64 = "$env:RUNTIME" -eq 'arm64' Install-Wix -arm64:$isArm64 } } diff --git a/tools/wix/wix.psm1 b/tools/wix/wix.psm1 index abc944aac9..4ab79adfc1 100644 --- a/tools/wix/wix.psm1 +++ b/tools/wix/wix.psm1 @@ -1,6 +1,27 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -Import-Module "$PSScriptRoot\dockerInstall.psm1" + +function Append-Path +{ + param + ( + $path + ) + $machinePathString = [System.Environment]::GetEnvironmentVariable('path',[System.EnvironmentVariableTarget]::Machine) + $machinePath = $machinePathString -split ';' + + if($machinePath -inotcontains $path) + { + $newPath = "$machinePathString;$path" + Write-Verbose "Adding $path to path..." -Verbose + [System.Environment]::SetEnvironmentVariable('path',$newPath,[System.EnvironmentVariableTarget]::Machine) + Write-Verbose "Added $path to path." -Verbose + } + else + { + Write-Verbose "$path already in path." -Verbose + } +} # Install using Wix Zip because the MSI requires an older version of dotnet # which was large and unstable in docker @@ -14,16 +35,48 @@ function Install-Wix Remove-Item $targetRoot -Recurse -Force } $binPath = Join-Path -Path $targetRoot -ChildPath 'bin' + Register-PSRepository -Name 'dotnet-eng' -SourceLocation "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" # keep version in sync with Microsoft.PowerShell.Packaging.csproj Save-Module -name Microsoft.Signed.Wix -RequiredVersion '3.14.1-8722.20240403.1' -path "$binPath/" $docExpandPath = Join-Path -Path $binPath -ChildPath 'doc' $sdkExpandPath = Join-Path -Path $binPath -ChildPath 'sdk' + $docTargetPath = Join-Path -Path $targetRoot -ChildPath 'doc' + + if (-not (Test-Path $docTargetPath)) { + $null = New-Item -ItemType Directory -Path $docTargetPath + Write-Verbose -Verbose "Created doc directory for WIX at $docTargetPath" + } + $sdkTargetPath = Join-Path -Path $targetRoot -ChildPath 'sdk' + + if (-not (Test-Path $sdkTargetPath)) { + $null = New-Item -ItemType Directory -Path $sdkTargetPath + Write-Verbose -Verbose "Created sdk directory for WIX at $sdkTargetPath" + } + + $binTargetPath = Join-Path -Path $targetRoot -ChildPath 'bin' + + if (-not (Test-Path $binTargetPath)) { + $null = New-Item -ItemType Directory -Path $binTargetPath + Write-Verbose -Verbose "Created bin directory for WIX at $binTargetPath" + } + + $x86TargetPath = Join-Path -Path $binPath -ChildPath 'x86' + + if (-not (Test-Path $x86TargetPath)) { + $null = New-Item -ItemType Directory -Path $x86TargetPath + Write-Verbose -Verbose "Created x86 directory for WIX at $x86TargetPath" + } + Write-Verbose "Fixing folder structure ..." -Verbose - Move-Item -Path $docExpandPath -Destination $docTargetPath - Move-Item -Path $sdkExpandPath -Destination $sdkTargetPath + Copy-Item -Path $docExpandPath -Destination $docTargetPath -Force + Copy-Item -Path $sdkExpandPath -Destination $sdkTargetPath -Force + Copy-Item -Path "$binPath\wix\3.14.1\tools\*" -Destination $binTargetPath -Force + Copy-Item -Path $x86ExpandPath -Destination $x86TargetPath -Force -Recurse -Verbose + Copy-Item -Path "$x86ExpandPath\burn.exe" -Destination "$x86TargetPath\burn.exe" -Force -Verbose + Append-Path -path $binPath Write-Verbose "Done installing WIX!" }