diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 2a765619e1..3792e295cd 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -7,7 +7,11 @@ $RepoRoot = (Resolve-Path -Path "$PSScriptRoot/../..").Path $packagingStrings = Import-PowerShellDataFile "$PSScriptRoot\packaging.strings.psd1" Import-Module "$PSScriptRoot\..\Xml" -ErrorAction Stop -Force $DebianDistributions = @("deb") -$RedhatDistributions = @("rh") +$RedhatDistributions = @("rh","cm1") +$AllDistributions = @() +$AllDistributions += $DebianDistributions +$AllDistributions += $RedhatDistributions +$AllDistributions += 'macOs' $script:netCoreRuntime = 'net7.0' $script:iconFileName = "Powershell_black_64.png" $script:iconPath = Join-Path -path $PSScriptRoot -ChildPath "../../assets/$iconFileName" -Resolve @@ -523,6 +527,7 @@ function Start-PSPackage { foreach ($Distro in $Script:RedhatDistributions) { $Arguments["Distribution"] = $Distro if ($PSCmdlet.ShouldProcess("Create RPM Package for $Distro")) { + Write-Verbose -Verbose "Creating RPM Package for $Distro" New-UnixPackage @Arguments } } @@ -1041,7 +1046,7 @@ function New-UnixPackage { # Build package try { if ($PSCmdlet.ShouldProcess("Create $type package")) { - Write-Log "Creating package with fpm..." + Write-Log "Creating package with fpm $Arguments..." try { $Output = Start-NativeExecution { fpm $Arguments } } @@ -1303,7 +1308,7 @@ function Get-FpmArguments "-t", $Type, "-s", "dir" ) - if ($Distribution -eq 'rh') { + if ($Distribution -in $script:RedHatDistributions) { $Arguments += @("--rpm-dist", $Distribution) $Arguments += @("--rpm-os", "linux") $Arguments += @("--license", "MIT") @@ -1348,13 +1353,29 @@ function Get-FpmArguments function Get-PackageDependencies { - param( - [String] - [ValidateSet('rh','deb','macOS')] - $Distribution - ) + [CmdletBinding()] + param() + DynamicParam { + # Add a dynamic parameter '-Distribution' when the specified package type is 'deb'. + # The '-Distribution' parameter can be used to indicate which Debian distro this pacakge is targeting. + $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" + $ParameterAttr.Mandatory = $true + $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $Script:AllDistributions + $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" + $Attributes.Add($ParameterAttr) > $null + $Attributes.Add($ValidateSetAttr) > $null + + $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("Distribution", [string], $Attributes) + $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary" + $Dict.Add("Distribution", $Parameter) > $null + return $Dict + } End { + if ($PSBoundParameters.ContainsKey('Distribution')) { + $Distribution = $PSBoundParameters['Distribution'] + } + # These should match those in the Dockerfiles, but exclude tools like Git, which, and curl $Dependencies = @() if ($Distribution -eq 'deb') { @@ -1373,6 +1394,22 @@ function Get-PackageDependencies "openssl-libs", "libicu" ) + } elseif ($Distribution -eq 'cm1') { + # Taken from the list here: + # https://github.com/dotnet/dotnet-docker/blob/d451d6e9427f58c8508f1297c862663a27eb609f/src/runtime-deps/6.0/cbl-mariner1.0/amd64/Dockerfile#L6 + $Dependencies = @( + "glibc" + "libgcc" + "krb5" + "libstdc++" + "zlib" + "icu" + "openssl-libs" + ) + } elseif ($Distribution -eq 'macOS') { + # do nothing + } else { + throw "Unknown distribution $Distribution" } return $Dependencies @@ -1423,7 +1460,7 @@ function New-AfterScripts Write-Verbose -Message "AfterScript Distribution: $Distribution" -Verbose - if ($Distribution -eq 'rh') { + if ($Distribution -in $script:RedHatDistributions) { $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $packagingStrings.RedHatAfterInstallScript -f "$Link", $Destination | Out-File -FilePath $AfterInstallScript -Encoding ascii @@ -4401,7 +4438,7 @@ function Invoke-AzDevOpsLinuxPackageCreation { } try { - Write-Verbose "Packaging '$BuildType'-LTS:$LTS for $ReleaseTag ..." -Verbose + Write-Verbose "Packaging '$BuildType'; LTS:$LTS for $ReleaseTag ..." -Verbose Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\psoptions.json" diff --git a/tools/packaging/releaseTests/sbom.tests.ps1 b/tools/packaging/releaseTests/sbom.tests.ps1 index 0e64e6c504..9469f68576 100644 --- a/tools/packaging/releaseTests/sbom.tests.ps1 +++ b/tools/packaging/releaseTests/sbom.tests.ps1 @@ -8,57 +8,102 @@ Describe "Verify SBOMs" { $missingFromPackageCases = @() $missingFromManifestCases = @() Write-Verbose "${env:PACKAGE_FOLDER}" -Verbose - Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | ForEach-Object { - Write-Verbose "Found $($_.Name)..." -Verbose - $testCases += @{ - FilePath = $_.FullName - Name = $_.Name + Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | + Where-Object { $_.Name -notlike 'powershell-symbols*' } | + ForEach-Object { + Write-Verbose "Found $($_.Name)..." -Verbose + $testCases += @{ + FilePath = $_.FullName + Name = $_.Name + Extension = $_.Extension + } + } + + if ($IsLinux) { + Get-ChildItem $env:PACKAGE_FOLDER -Filter *.rpm | ForEach-Object { + Write-Verbose "Found $($_.Name)..." -Verbose + $testCases += @{ + FilePath = $_.FullName + Name = $_.Name + Extension = $_.Extension + } } } foreach($case in $testCases) { + $skip = $null $name = $case.Name Write-Verbose "Testing $name..." -Verbose $extractedPath = Join-Path Testdrive:\ -ChildPath ([System.io.path]::GetRandomFileName()) $null = New-Item -Path $extractedPath -ItemType Directory -Force - Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath - $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json' + $resolvedPath = (Resolve-Path -Path $extractedPath).ProviderPath + switch ($case.Extension) { + '.zip' { + Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath + $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json' + } + '.rpm' { + Push-Location $resolvedPath + try { + if (Get-Command -Name rpm2cpio -ErrorAction SilentlyContinue) { + rpm2cpio $case.FilePath | cpio -i --make-directories 2>cpio.log + $manifestPath = Get-ChildItem -Path manifest.spdx.json -Recurse | Select-Object -First 1 -ExpandProperty FullName + $extractedPath = Get-ChildItem -Path system.management.automation.dll -Recurse | Select-Object -First 1 -ExpandProperty DirectoryName + } else { + $skip = "rpm2cpio not found" + } + } finally { + Pop-Location + } + } + Default { + throw "Unkown extension $($case.Extension)" + } + } + It "$name has a BOM" { + if ($skip) { + Set-ItResult -Pending -Because $skip + } $manifestPath | Should -Exist } - Test-PackageManifest -PackagePath $extractedPath | ForEach-Object { - $status = $_.Status - $expectedHash = $_.ExpectedHash - $actual = $_.ActualHash - $file = $_.File - switch($status) { - # cover match and mismatch - default { - $matchCases += @{ - Name = $name - File = $file - ActualHash = $actual - ExpectedHash = $ExpectedHash - Status = $status + # RPM hashes are broken, skip that + if ($case.Extension -in '.zip') { + Test-PackageManifest -PackagePath $extractedPath | ForEach-Object { + $status = $_.Status + $expectedHash = $_.ExpectedHash + $actual = $_.ActualHash + $file = $_.File + + switch ($status) { + # cover match and mismatch + default { + $matchCases += @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } } - } - "MissingFromPackage" { - $missingFromPackageCases = @{ - Name = $name - File = $file - ActualHash = $actual - ExpectedHash = $ExpectedHash - Status = $status + "MissingFromPackage" { + $missingFromPackageCases = @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } } - } - "MissingFromManifest" { - $missingFromManifestCases = @{ - Name = $name - File = $file - ActualHash = $actual - ExpectedHash = $ExpectedHash - Status = $status + "MissingFromManifest" { + $missingFromManifestCases = @{ + Name = $name + File = $file + ActualHash = $actual + ExpectedHash = $ExpectedHash + Status = $status + } } } } @@ -66,11 +111,7 @@ Describe "Verify SBOMs" { } } - Context "Zip files" { - BeforeAll { - Write-Verbose "In Context BeforeAll" -Verbose - } - + Context "Package files" { It " should have with matching hash" -TestCases $matchCases { param( $Name, diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index c403aff28a..e1334f9320 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -122,9 +122,7 @@ stages: displayName: Validate Package BOM pool: # testing - vmImage: windows-latest - variables: - - group: 'Azure Blob variable group' + vmImage: ubuntu-latest steps: - template: templates/release-ValidatePackageBOM.yml diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 3d9625201a..6dd44eb219 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -61,12 +61,16 @@ jobs: condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) workingDirectory: $(PowerShellRoot) - - powershell: | + - pwsh: | + try { + Import-Module "$env:POWERSHELLROOT/build.psm1" + Import-Module "$env:POWERSHELLROOT/tools/packaging" - Import-Module "$env:POWERSHELLROOT/build.psm1" - Import-Module "$env:POWERSHELLROOT/tools/packaging" - - Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + } catch { + Get-Error + throw + } displayName: 'Build' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) workingDirectory: $(PowerShellRoot) @@ -111,20 +115,25 @@ jobs: sourceScanPath: '$(PowerShellRoot)/tools' - powershell: | - Import-Module "$env:POWERSHELLROOT/build.psm1" - Import-Module "$env:POWERSHELLROOT/tools/packaging" + try { + Import-Module "$env:POWERSHELLROOT/build.psm1" + Import-Module "$env:POWERSHELLROOT/tools/packaging" - $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json + $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json - # LTSRelease.Package indicates that the release should be packaged as an LTS - $LTS = $metadata.LTSRelease.Package - Write-Verbose -Verbose -Message "LTS is set to: $LTS" + # LTSRelease.Package indicates that the release should be packaged as an LTS + $LTS = $metadata.LTSRelease.Package + Write-Verbose -Verbose -Message "LTS is set to: $LTS" - Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' - if ($LTS) { - Write-Verbose -Verbose "Packaging LTS" - Invoke-AzDevOpsLinuxPackageCreation -LTS -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + if ($LTS) { + Write-Verbose -Verbose "Packaging LTS" + Invoke-AzDevOpsLinuxPackageCreation -LTS -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)' + } + } catch { + Get-Error + throw } displayName: 'Package' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) @@ -244,10 +253,25 @@ jobs: signOutputPath: $(Build.StagingDirectory)\signedPackages certificateId: "CP-450779-Pgp" pattern: | - **\*.rpm + **\*.rh.*.rpm useMinimatch: true shouldSign: $(SHOULD_SIGN) - displayName: Sign RPM + displayName: Sign RedHat RPM + OutputMode: AlwaysCopy + + - ${{ if eq(variables['buildName'], 'RPM') }}: + - template: EsrpSign.yml@ComplianceRepo + parameters: + # Sign in-place, previous task copied the files to this folder + buildOutputPath: $(Build.StagingDirectory)\signedPackages + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-459159-Pgp" + pattern: | + **\*.cm1.*.rpm + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + displayName: Sign Mariner RPM + OutputMode: NeverCopy # requires windows - ${{ if ne(variables['buildName'], 'RPM') }}: diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index ef16761f70..69c683156d 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -109,13 +109,23 @@ jobs: displayName: Compress signed files - pwsh: | - tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap + try { + tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap + } catch { + Get-Error + throw + } displayName: 'Bootstrap VM' - pwsh: | # Add -SkipReleaseChecks as a mitigation to unblock release. # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks + try { + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks + } catch { + Get-Error + throw + } displayName: 'Package' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml index 4778ebcaa1..3fd560cbd0 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml @@ -14,17 +14,17 @@ steps: Write-Host "##vso[build.updatebuildnumber]$name" displayName: Set Release Name -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - Install-AzCopy - displayName: Install AzCopy +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: PowerShellCore + pipeline: '696' + preferTriggeringPipeline: true + runVersion: latestFromBranch + runBranch: '$(Build.SourceBranch)' + artifact: finalResults + path: $(System.ArtifactsDirectory) -- pwsh: | - Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' - $azcopy = Find-AzCopy - - & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) - displayName: Download Azure Artifacts - pwsh: | Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml index fa80c1d0c8..d50c94bcba 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml @@ -30,7 +30,7 @@ steps: - pwsh: | $message = @() Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.rpm | ForEach-Object { - if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.rh.x86_64\.rpm') + if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.(rh|cm1).x86_64\.rpm') { $messageInstance = "$($_.Name) is not a valid package name" $message += $messageInstance