diff --git a/.config/suppress.json b/.config/suppress.json new file mode 100644 index 0000000000..55e607b1b0 --- /dev/null +++ b/.config/suppress.json @@ -0,0 +1,17 @@ +{ + "tool": "Credential Scanner", + "suppressions": [ + { + "file": "\\test\\tools\\Modules\\WebListener\\ClientCert.pfx", + "_justification": "Test certificate with private key" + }, + { + "file": "\\test\\tools\\Modules\\WebListener\\ServerCert.pfx", + "_justification": "Test certificate with private key" + }, + { + "file": "\\test\\powershell\\Modules\\Microsoft.PowerShell.Security\\certificateCommon.psm1", + "_justification": "Test certificate with private key and inline suppression isn't working" + } + ] +} diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json new file mode 100644 index 0000000000..90d4db3675 --- /dev/null +++ b/.config/tsaoptions.json @@ -0,0 +1,5 @@ +{ + "instanceUrl": "https://msazure.visualstudio.com", + "projectName": "One", + "areaPath": "One\\MGMT\\Compute\\Powershell\\Powershell\\PowerShell Core" +} diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml new file mode 100644 index 0000000000..432a627b0d --- /dev/null +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -0,0 +1,251 @@ +name: UnifiedPackageBuild-$(Build.BuildId) +trigger: + branches: + include: + - master + - release* +pr: + branches: + include: + - master + - release* + +parameters: + - name: ForceAzureBlobDelete + displayName: Delete Azure Blob + type: string + values: + - true + - false + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +variables: + - name: PS_RELEASE_BUILD + value: 1 + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: branchCounterKey + value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] + - name: branchCounter + value: $[counter(variables['branchCounterKey'], 1)] + - name: ForceAzureBlobDelete + value: ${{ parameters.ForceAzureBlobDelete }} + - name: BUILDSECMON_OPT_IN + value: true + - name: __DOTNET_RUNTIME_FEED + value: ${{ parameters.InternalSDKBlobURL }} + - name: LinuxContainerImage + value: onebranch.azurecr.io/linux/ubuntu-2004:latest + - name: WindowsContainerImage + value: onebranch.azurecr.io/windows/ltsc2019/vse2022:latest + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - name: SKIP_SIGNING + value: ${{ parameters.SKIP_SIGNING }} + - group: 'AzDevOpsArtifacts' + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates + parameters: + customTags: 'ES365AIMigrationTooling' + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + enabled: true + ${{ else }}: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + asyncSdl: # https://aka.ms/obpipelines/asyncsdl + enabled: true + forStages: [prep, macos, linux, windows, SignFiles, test_and_release_artifacts] + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + binskim: + enabled: false + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json + + stages: + - stage: prep + jobs: + - template: /.pipelines/templates/checkAzureContainer.yml@self + + - stage: macos + displayName: macOS - build and sign + dependsOn: ['prep'] + jobs: + - template: /.pipelines/templates/mac.yml@self + parameters: + buildArchitecture: x64 + - template: /.pipelines/templates/mac.yml@self + parameters: + buildArchitecture: arm64 + + - stage: linux + displayName: linux - build and sign + dependsOn: ['prep'] + jobs: + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64_minSize' + BuildConfiguration: 'minSize' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-arm' + JobName: 'linux_arm' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-arm64' + JobName: 'linux_arm64' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-x64' + JobName: 'linux_fxd_x64_mariner' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-arm64' + JobName: 'linux_fxd_arm64_mariner' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-noopt-linux-musl-x64' + JobName: 'linux_fxd_x64_alpine' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent' + JobName: 'linux_fxd' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-musl-x64' + JobName: 'linux_x64_alpine' + + - stage: windows + displayName: windows - build and sign + dependsOn: ['prep'] + jobs: + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: release + JobName: build_windows_x64_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: minSize + JobName: build_windows_x64_minSize + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x86 + JobName: build_windows_x86_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: arm64 + JobName: build_windows_arm64_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependent + JobName: build_windows_fxdependent_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependentWinDesktop + JobName: build_windows_fxdependentWinDesktop_release + + - stage: test_and_release_artifacts + displayName: Test and Release Artifacts + dependsOn: ['prep'] + jobs: + - template: /.pipelines/templates/testartifacts.yml@self + + - job: release_json + displayName: Create and Upload release.json + pool: + type: windows + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + steps: + - checkout: self + clean: true + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + - powershell: | + $metadata = Get-Content '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease.Package + @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" + Get-Content "$(Build.StagingDirectory)\release.json" + + if (-not (Test-Path "$(ob_outputDirectory)\metadata")) { + New-Item -ItemType Directory -Path "$(ob_outputDirectory)\metadata" + } + + Copy-Item -Path "$(Build.StagingDirectory)\release.json" -Destination "$(ob_outputDirectory)\metadata" -Force + displayName: Create and upload release.json file to build artifact + retryCountOnTaskFailure: 2 + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/SetVersionVariables.yml b/.pipelines/templates/SetVersionVariables.yml new file mode 100644 index 0000000000..f8fb68c10b --- /dev/null +++ b/.pipelines/templates/SetVersionVariables.yml @@ -0,0 +1,68 @@ +parameters: + ReleaseTagVar: v6.2.0 + ReleaseTagVarName: ReleaseTagVar + CreateJson: 'no' + UseJson: 'yes' + +steps: +- ${{ if eq(parameters['UseJson'],'yes') }}: + - task: DownloadBuildArtifacts@0 + inputs: + artifactName: 'drop_prep_DeleteBlob' + itemPattern: '*.json' + downloadPath: '$(System.ArtifactsDirectory)' + displayName: Download Build Info Json + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + +- powershell: | + $path = "./build.psm1" + if($env:REPOROOT){ + Write-Verbose "reporoot already set to ${env:REPOROOT}" -Verbose + exit 0 + } + if(Test-Path -Path $path) + { + Write-Verbose "reporoot detect at: ." -Verbose + $repoRoot = '.' + } + else{ + $path = "./PowerShell/build.psm1" + if(Test-Path -Path $path) + { + Write-Verbose "reporoot detect at: ./PowerShell" -Verbose + $repoRoot = './PowerShell' + } + } + if($repoRoot) { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]$repoRoot" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } else { + Write-Verbose -Verbose "repo not found" + } + displayName: 'Set repo Root' + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + +- powershell: | + $createJson = ("${{ parameters.CreateJson }}" -ne "no") + $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson + $version = $releaseTag.Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + $azureVersion = $releaseTag.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: 'Set ${{ parameters.ReleaseTagVarName }} and other version Variables' + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + +- powershell: | + Get-ChildItem -Path env: + displayName: Capture environment + condition: succeededOrFailed() + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase diff --git a/.pipelines/templates/checkAzureContainer.yml b/.pipelines/templates/checkAzureContainer.yml new file mode 100644 index 0000000000..d761d9f327 --- /dev/null +++ b/.pipelines/templates/checkAzureContainer.yml @@ -0,0 +1,90 @@ +jobs: +- job: DeleteBlob + variables: + - group: Azure Blob variable group + - group: AzureBlobServiceConnection + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' + - name: ob_sdl_sbom_enabled + value: false + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + + displayName: Delete blob is exists + pool: + type: windows + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + + - pwsh: | + if (-not (Test-Path -Path $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json)) { + Get-ChildItem -Path $(Build.SourcesDirectory) -Recurse + throw 'tsaoptions.json not found' + } + displayName: 'Check tsaoptions.json' + + - pwsh: | + if (-not (Test-Path -Path $(Build.SourcesDirectory)\PowerShell\.config\suppress.json)) { + Get-ChildItem -Path $(Build.SourcesDirectory) -Recurse + throw 'suppress.json not found' + } + displayName: 'Check suppress.json' + + # Needed as per FAQ here: https://eng.ms/docs/products/onebranch/build/troubleshootingfaqs + - task: PowerShell@2 + displayName: 'Update Az.Storage Module' + inputs: + targetType: 'inline' + script: | + Get-PackageProvider -Name NuGet -ForceBootstrap + Install-Module -Name Az.Storage -Verbose -Force -AllowClobber + Uninstall-AzureRm -Verbose + + - task: AzurePowerShell@5 + displayName: Check if blob exists and delete if specified + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: latestVersion + inline: | + try { + $container = Get-AzStorageContainer -Container '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -ErrorAction Stop + if ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'false') { + throw 'Azure blob container $(AzureVersion) already exists. To overwrite, use ForceAzureBlobDelete parameter' + } + elseif ($container -ne $null -and '$(ForceAzureBlobDelete)' -eq 'true') { + Write-Verbose -Verbose 'Removing container $(AzureVersion) due to ForceAzureBlobDelete parameter' + Remove-AzStorageContainer -Name '$(AzureVersion)' -Context (New-AzStorageContext -StorageAccountName '$(StorageAccount)') -Force + } + } + catch { + if ($_.FullyQualifiedErrorId -eq 'ResourceNotFoundException,Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet.GetAzureStorageContainerCommand') { + Write-Verbose -Verbose 'Container "$(AzureVersion)" does not exists.' + } + else { + throw $_ + } + } + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/cloneToOfficialPath.yml b/.pipelines/templates/cloneToOfficialPath.yml new file mode 100644 index 0000000000..844d8b8028 --- /dev/null +++ b/.pipelines/templates/cloneToOfficialPath.yml @@ -0,0 +1,19 @@ +parameters: + nativePathRoot: '' + +steps: +- powershell: | + $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar + $nativePath = "${{parameters.nativePathRoot }}${dirSeparatorChar}PowerShell" + Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" + if ((Test-Path "$nativePath")) { + Remove-Item -Path "$nativePath" -Force -Recurse -Verbose -ErrorAction ignore + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + git clone --quiet $env:REPOROOT $nativePath + displayName: Clone PowerShell Repo to /PowerShell + errorActionPreference: silentlycontinue + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase diff --git a/.pipelines/templates/insert-nuget-config-azfeed.yml b/.pipelines/templates/insert-nuget-config-azfeed.yml new file mode 100644 index 0000000000..2279c4839a --- /dev/null +++ b/.pipelines/templates/insert-nuget-config-azfeed.yml @@ -0,0 +1,33 @@ +parameters: +- name: "repoRoot" + default: $(REPOROOT) +steps: +- pwsh: | + $configPath = "${env:NugetConfigDir}/nuget.config" + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" + if(-not (Test-Path $configPath)) + { + throw "nuget.config is not created" + } + Get-Content $configPath | Write-Verbose -Verbose + displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules' + condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], '')) + env: + NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + +- pwsh: | + $configPath = "${env:NugetConfigDir}/nuget.config" + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" + if(-not (Test-Path $configPath)) + { + throw "nuget.config is not created" + } + Get-Content $configPath | Write-Verbose -Verbose + displayName: 'Add nuget.config for Azure DevOps feed for packages' + condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], '')) + env: + NugetConfigDir: ${{ parameters.repoRoot }} + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml new file mode 100644 index 0000000000..134d581833 --- /dev/null +++ b/.pipelines/templates/linux.yml @@ -0,0 +1,183 @@ +parameters: + Runtime: 'linux-x64' + BuildConfiguration: 'release' + JobName: 'build_linux' + +jobs: +- job: build_${{ parameters.JobName }} + displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + pool: + type: linux + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: Runtime + value: ${{ parameters.Runtime }} + - name: ob_sdl_sbom_packageName + value: 'Microsoft.Powershell.Linux.${{ parameters.Runtime }}' + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + inputs: + Enabled: true + AnalyzeInPipeline: true + Language: csharp + + - pwsh: | + $runtime = $env:RUNTIME + + $params = @{} + if ($env:BuildConfiguration -eq 'minSize') { + $params['ForMinimalSize'] = $true + } + + Write-Verbose -Message "Building PowerShell with Runtime: $runtime" + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Runtime) -Force + + Start-PSBootstrap + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath @params -Clean -PSModuleRestore + + Write-Verbose -Verbose "Verifying pdbs exist in build folder" + $pdbs = Get-ChildItem -Path $buildWithSymbolsPath -Recurse -Filter *.pdb + if ($pdbs.Count -eq 0) { + Write-Error -Message "No pdbs found in build folder" + } + else { + Write-Verbose -Verbose "Found $($pdbs.Count) pdbs in build folder" + $pdbs | ForEach-Object { + Write-Verbose -Verbose "Pdb: $($_.FullName)" + } + } + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" + displayName: 'Build Linux - $(Runtime)' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - pwsh: | + $platform = 'linux' + $vstsCommandString = "vso[task.setvariable variable=ArtifactPlatform]$platform" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set artifact platform + + - pwsh: | + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Unsigned-$(Runtime)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '$(Pipeline.Workspace)/Symbols_$(Runtime)/*' -Destination $pathForUpload -Recurse -Force -Verbose + displayName: Copy unsigned files for upload + + - template: /.pipelines/templates/step/finalize.yml@self + +- job: sign_${{ parameters.JobName }} + displayName: Sign_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + dependsOn: build_${{ parameters.JobName }} + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: Runtime + value: ${{ parameters.Runtime }} + - name: ob_sdl_codeql_compiled_enabled + value: false + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: drop_linux_build_${{ parameters.JobName }} + path: $(Pipeline.Workspace)/drop_linux_build + displayName: Download build + + - pwsh: | + Get-ChildItem -Path $(Pipeline.Workspace)/drop_linux_build -Recurse + displayName: Capture downloaded files + + - pwsh: | + $pwshPath = Get-ChildItem -Path $(Pipeline.Workspace)/drop_linux_build -File -Recurse | Where-Object { $_.Name -eq 'pwsh' } + $rootPath = Split-Path -Path $pwshPath.FullName -Parent + Write-Verbose -Verbose "Setting vso[task.setvariable variable=DropRootPath]$rootPath" + Write-Host "##vso[task.setvariable variable=DropRootPath]$rootPath" + displayName: Set drop root path + + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: $(DropRootPath) + + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/mac.yml b/.pipelines/templates/mac.yml new file mode 100644 index 0000000000..7542ffad4c --- /dev/null +++ b/.pipelines/templates/mac.yml @@ -0,0 +1,132 @@ +parameters: + buildArchitecture: 'x64' +jobs: +- job: build_macOS_${{ parameters.buildArchitecture }} + displayName: Build macOS ${{ parameters.buildArchitecture }} + condition: succeeded() + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-latest' + + variables: + - name: HOMEBREW_NO_ANALYTICS + value: 1 + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_credscan_suppressionsfileforartifacts + value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + - pwsh: | + # create folder + sudo mkdir "$(Agent.TempDirectory)/PowerShell" + # make the current user the owner + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + parameters: + nativePathRoot: '$(Agent.TempDirectory)' + - pwsh: | + tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap + displayName: 'Bootstrap VM' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + - pwsh: | + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks + $env:AzDevOpsFeedPAT2 = $null + displayName: 'Build' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - template: /.pipelines/templates/step/finalize.yml@self + +- job: sign_${{ parameters.buildArchitecture }} + displayName: Sign_macOS_${{ parameters.buildArchitecture }} + condition: succeeded() + dependsOn: build_macOS_${{ parameters.buildArchitecture }} + pool: + type: windows + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: true + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: BuildArchitecture + value: ${{ parameters.buildArchitecture }} + - name: ob_sdl_codeql_compiled_enabled + value: false + - name: ob_sdl_sbom_packageName + value: 'Microsoft.Powershell.Windows.${{parameters.buildArchitecture}}' + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: 'macosBinResults' + path: '$(Pipeline.Workspace)\Symbols' + displayName: Download build + + - pwsh: | + Get-ChildItem "$(Pipeline.Workspace)\*" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + $runtime = '$(BuildArchitecture)' + Write-Host "sending.. vso[task.setvariable variable=Runtime]$runtime" + Write-Host "##vso[task.setvariable variable=Runtime]$runtime" + + $zipPath = Get-Item '$(Pipeline.Workspace)\Symbols\*symbol*${{ parameters.buildArchitecture }}*.zip' -Verbose + Write-Verbose -Verbose "Zip Path: $zipPath" + + $expandedFolder = $zipPath.BaseName + Expand-Archive -Path $zipPath -Destination "$(Pipeline.Workspace)\$expandedFolder" -Force + $rootPath = "$(Pipeline.Workspace)\$expandedFolder" + Write-Verbose -Verbose "Setting vso[task.setvariable variable=DropRootPath]$rootPath" + Write-Host "##vso[task.setvariable variable=DropRootPath]$rootPath" + displayName: Expand symbols zip + + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: $(DropRootPath) + + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/obp-file-signing.yml b/.pipelines/templates/obp-file-signing.yml new file mode 100644 index 0000000000..06cd46dec2 --- /dev/null +++ b/.pipelines/templates/obp-file-signing.yml @@ -0,0 +1,152 @@ +parameters: + binPath: '$(ob_outputDirectory)' + +steps: +- pwsh: | + $fullSymbolsFolder = '${{ parameters.binPath }}' + Write-Verbose -Verbose "fullSymbolsFolder == $fullSymbolsFolder" + Get-ChildItem -Recurse $fullSymbolsFolder | Select-Object -ExpandProperty FullName | Write-Verbose -Verbose + $filesToSignDirectory = "$(Pipeline.Workspace)/toBeSigned" + if ((Test-Path -Path $filesToSignDirectory)) { + Remove-Item -Path $filesToSignDirectory -Recurse -Force + } + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $itemsToCopyWithRecurse = @( + "$($fullSymbolsFolder)/*.ps1" + "$($fullSymbolsFolder)/Microsoft.PowerShell*.dll" + ) + $itemsToCopy = @{ + "$($fullSymbolsFolder)/*.ps1" = "" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Host/Microsoft.PowerShell.Host.psd1" = "Modules/Microsoft.PowerShell.Host" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1" = "Modules/Microsoft.PowerShell.Management" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Security/Microsoft.PowerShell.Security.psd1" = "Modules/Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1" = "Modules/Microsoft.PowerShell.Utility" + "$($fullSymbolsFolder)/pwsh.dll" = "" + "$($fullSymbolsFolder)/System.Management.Automation.dll" = "" + } + ## Windows only modules + if('$(ArtifactPlatform)' -eq 'windows') { + $itemsToCopy += @{ + "$($fullSymbolsFolder)/pwsh.exe" = "" + "$($fullSymbolsFolder)/Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" + "$($fullSymbolsFolder)/Microsoft.WSMan.*.dll" = "" + "$($fullSymbolsFolder)/Modules/CimCmdlets/CimCmdlets.psd1" = "Modules/CimCmdlets" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Diagnostics.format.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Event.format.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/GetEvent.types.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Security/Security.types.ps1xml" = "Modules/Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Microsoft.PowerShell.Diagnostics.psd1" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.WSMan.Management/Microsoft.WSMan.Management.psd1" = "Modules/Microsoft.WSMan.Management" + "$($fullSymbolsFolder)/Modules/Microsoft.WSMan.Management/WSMan.format.ps1xml" = "Modules/Microsoft.WSMan.Management" + "$($fullSymbolsFolder)/Modules/PSDiagnostics/PSDiagnostics.ps?1" = "Modules/PSDiagnostics" + } + } + + $itemsToExclude = @( + # This package is retrieved from https://www.github.com/powershell/MarkdownRender + "$($fullSymbolsFolder)/Microsoft.PowerShell.MarkdownRender.dll" + ) + + if('$(ArtifactPlatform)' -eq 'linux' -or '$(ArtifactPlatform)' -eq 'macos') { + $itemsToExclude += "$($fullSymbolsFolder)/pwsh" + } + + Write-Verbose -verbose "recursively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude + Write-Verbose -verbose "recursive copy done." + + foreach($pattern in $itemsToCopy.Keys) { + $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern + $null = New-Item -ItemType Directory -Path $destinationFolder -Force + Write-Verbose -verbose "copying $pattern to $destinationFolder" + + if (-not (Test-Path -Path $pattern)) { + Write-Verbose -verbose "No files found for pattern $pattern" + continue + } + + Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose + } + + Write-Verbose -verbose "copying done." + Write-Verbose -verbose "Files to be signed at: $filesToSignDirectory" + + Get-ChildItem -Recurse -File $filesToSignDirectory | Select-Object -Property FullName + displayName: 'Prepare files to be signed' + +- task: onebranch.pipeline.signing@1 + displayName: Sign 1st party files + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' + search_root: $(Pipeline.Workspace)/toBeSigned + +- pwsh : | + Get-ChildItem -Path env: + displayName: Capture environment + +- pwsh: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + + $BuildPath = (Get-Item '${{ parameters.binPath }}').FullName + Write-Verbose -Verbose -Message "BuildPath: $BuildPath" + + ## copy all files to be signed to build folder + Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath '$(Pipeline.Workspace)/toBeSigned' + + $dlls = Get-ChildItem $BuildPath/*.dll, $BuildPath/*.exe -Recurse + $signatures = $dlls | Get-AuthenticodeSignature + $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch '^CN=Microsoft.*'}| select-object -ExpandProperty Path + + Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" + + $filesToSignDirectory = "$(Pipeline.Workspace)/thirdPartyToBeSigned" + if (Test-Path $filesToSignDirectory) { + Remove-Item -Path $filesToSignDirectory -Recurse -Force + } + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force -Verbose + + $missingSignatures | ForEach-Object { + $pathWithoutLeaf = Split-Path $_ + $relativePath = $pathWithoutLeaf.replace($BuildPath,'') + Write-Verbose -Verbose -Message "relativePath: $relativePath" + $targetDirectory = Join-Path -Path $filesToSignDirectory -ChildPath $relativePath + Write-Verbose -Verbose -Message "targetDirectory: $targetDirectory" + if(!(Test-Path $targetDirectory)) + { + $null = New-Item -ItemType Directory -Path $targetDirectory -Force -Verbose + } + Copy-Item -Path $_ -Destination $targetDirectory + } + displayName: Create ThirdParty Signing Folder + +- task: onebranch.pipeline.signing@1 + displayName: Sign 3rd Party files + inputs: + command: 'sign' + signing_profile: 135020002 + files_to_sign: '**\*.dll;**\*.exe' + search_root: $(Pipeline.Workspace)/thirdPartyToBeSigned + +- pwsh: | + Get-ChildItem '$(Pipeline.Workspace)/thirdPartyToBeSigned/*' + displayName: Capture ThirdParty Signed files + +- pwsh: | + Import-Module '$(PowerShellRoot)/build.psm1' -Force + Import-Module '$(PowerShellRoot)/tools/packaging' -Force + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Signed-$(Runtime)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose + Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + + Write-Verbose "Copying third party signed files to the build folder" + $thirdPartySignedFilesPath = (Get-Item '$(Pipeline.Workspace)/thirdPartyToBeSigned').FullName + Update-PSSignedBuildFolder -BuildPath $pathForUpload -SignedFilesPath $thirdPartySignedFilesPath + + displayName: 'Copy signed files for upload' + +- template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/shouldSign.yml b/.pipelines/templates/shouldSign.yml new file mode 100644 index 0000000000..3f02489894 --- /dev/null +++ b/.pipelines/templates/shouldSign.yml @@ -0,0 +1,23 @@ +steps: +- powershell: | + $shouldSign = $true + $authenticodeCert = 'CP-230012' + $msixCert = 'CP-230012' + if($env:IS_DAILY -eq 'true') + { + $authenticodeCert = 'CP-460906' + } + if($env:SKIP_SIGNING -eq 'Yes') + { + $shouldSign = $false + } + $vstsCommandString = "vso[task.setvariable variable=SHOULD_SIGN]$($shouldSign.ToString().ToLowerInvariant())" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + $vstsCommandString = "vso[task.setvariable variable=MSIX_CERT]$($msixCert)" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + $vstsCommandString = "vso[task.setvariable variable=AUTHENTICODE_CERT]$($authenticodeCert)" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Set SHOULD_SIGN Variable' diff --git a/.pipelines/templates/step/finalize.yml b/.pipelines/templates/step/finalize.yml new file mode 100644 index 0000000000..78e0341c82 --- /dev/null +++ b/.pipelines/templates/step/finalize.yml @@ -0,0 +1,6 @@ +# This was used before migrating to OneBranch to deal with one of the SDL taks from failing with a warning instead of an error. +steps: +- pwsh: | + throw "Jobs with an Issue will not work for release. Please fix the issue and try again." + displayName: Check for SucceededWithIssues + condition: eq(variables['Agent.JobStatus'],'SucceededWithIssues') diff --git a/.pipelines/templates/testartifacts.yml b/.pipelines/templates/testartifacts.yml new file mode 100644 index 0000000000..883d3cd21c --- /dev/null +++ b/.pipelines/templates/testartifacts.yml @@ -0,0 +1,108 @@ +jobs: +- job: build_testartifacts_win + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + displayName: Build windows test artifacts + condition: succeeded() + pool: + type: windows + steps: + - checkout: self + clean: true + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(Build.SourcesDirectory) + - pwsh: | + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + Start-PSBootstrap + displayName: Bootstrap + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - pwsh: | + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + function BuildTestPackage([string] $runtime) + { + Write-Verbose -Verbose "Starting to build package for $runtime" + New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime + if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) + { + throw "Test Package was not found at: $(System.ArtifactsDirectory)" + } + switch ($runtime) + { + win7-x64 { $packageName = "TestPackage-win-x64.zip" } + win7-x86 { $packageName = "TestPackage-win-x86.zip" } + win-arm64 { $packageName = "TestPackage-win-arm64.zip" } + } + Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName + Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" + } + BuildTestPackage -runtime win7-x64 + BuildTestPackage -runtime win7-x86 + BuildTestPackage -runtime win-arm64 + displayName: Build test package and upload + retryCountOnTaskFailure: 1 +- job: build_testartifacts_nonwin + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + displayName: Build non-windows test artifacts + condition: succeeded() + pool: + type: linux + steps: + - checkout: self + clean: true + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(Build.SourcesDirectory) + - pwsh: | + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + Start-PSBootstrap + displayName: Bootstrap + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + - pwsh: | + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + function BuildTestPackage([string] $runtime) + { + Write-Verbose -Verbose "Starting to build package for $runtime" + New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime + if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) + { + throw "Test Package was not found at: $(System.ArtifactsDirectory)" + } + switch ($runtime) + { + linux-x64 { $packageName = "TestPackage-linux-x64.zip" } + linux-arm { $packageName = "TestPackage-linux-arm.zip" } + linux-arm64 { $packageName = "TestPackage-linux-arm64.zip" } + osx-x64 { $packageName = "TestPackage-macOS.zip" } + linux-musl-x64 { $packageName = "TestPackage-alpine-x64.zip"} + } + Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName + Write-Host "##vso[artifact.upload containerfolder=testArtifacts;artifactname=testArtifacts]$(System.ArtifactsDirectory)/$packageName" + } + BuildTestPackage -runtime linux-x64 + BuildTestPackage -runtime linux-arm + BuildTestPackage -runtime linux-arm64 + BuildTestPackage -runtime osx-x64 + BuildTestPackage -runtime linux-musl-x64 + displayName: Build test package and upload + retryCountOnTaskFailure: 1 + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml new file mode 100644 index 0000000000..1f100783a2 --- /dev/null +++ b/.pipelines/templates/windows-hosted-build.yml @@ -0,0 +1,126 @@ +parameters: + Architecture: 'x64' + BuildConfiguration: 'release' + JobName: 'build_windows' + +jobs: +- job: build_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + displayName: Build_Windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: Architecture + value: ${{ parameters.Architecture }} + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: ob_sdl_sbom_packageName + value: 'Microsoft.Powershell.Windows.${{ parameters.Architecture }}' + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + inputs: + Enabled: true + AnalyzeInPipeline: true + Language: csharp + + - pwsh: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + $params = @{} + if ($env:BuildConfiguration -eq 'minSize') { + $params['ForMinimalSize'] = $true + } + + $vstsCommandString = "vso[task.setvariable variable=Runtime]$runtime" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + Write-Verbose -Message "Building PowerShell with Runtime: $runtime for '$env:BuildConfiguration' configuration" + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Architecture) -Force + + Start-PSBootstrap -Package + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params + + Write-Verbose -Verbose "Verifying pdbs exist in build folder" + $pdbs = Get-ChildItem -Path $buildWithSymbolsPath -Recurse -Filter *.pdb + if ($pdbs.Count -eq 0) { + Write-Error -Message "No pdbs found in build folder" + } + else { + Write-Verbose -Verbose "Found $($pdbs.Count) pdbs in build folder" + $pdbs | ForEach-Object { + Write-Verbose -Verbose "Pdb: $($_.FullName)" + } + } + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" + displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols folder' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - pwsh: | + $platform = 'windows' + $vstsCommandString = "vso[task.setvariable variable=ArtifactPlatform]$platform" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set artifact platform + + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' + + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index e2c3a128b5..169d26ae3a 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -763,17 +763,54 @@ function Update-PSSignedBuildFolder [string[]] $RemoveFilter = ('*.pdb', '*.zip', '*.r2rmap') ) + $BuildPathNormalized = (Get-Item $BuildPath).FullName + $SignedFilesPathNormalized = (Get-Item $SignedFilesPath).FullName + + Write-Verbose -Verbose "BuildPath = $BuildPathNormalized" + Write-Verbose -Verbose "SignedFilesPath = $signedFilesPath" + # Replace unsigned binaries with signed - $signedFilesFilter = Join-Path -Path $SignedFilesPath -ChildPath '*' + $signedFilesFilter = Join-Path -Path $SignedFilesPathNormalized -ChildPath '*' + Write-Verbose -Verbose "signedFilesFilter = $signedFilesFilter" + Get-ChildItem -Path $signedFilesFilter -Recurse -File | Select-Object -ExpandProperty FullName | ForEach-Object -Process { - $relativePath = $_.ToLowerInvariant().Replace($SignedFilesPath.ToLowerInvariant(),'') - $destination = Join-Path -Path $BuildPath -ChildPath $relativePath + Write-Verbose -Verbose "Processing $_" + + # Agents seems to be on a case sensitive file system + if ($IsLinux) { + $relativePath = $_.Replace($SignedFilesPathNormalized, '') + } else { + $relativePath = $_.ToLowerInvariant().Replace($SignedFilesPathNormalized.ToLowerInvariant(), '') + } + + Write-Verbose -Verbose "relativePath = $relativePath" + $destination = (Get-Item (Join-Path -Path $BuildPathNormalized -ChildPath $relativePath)).FullName + Write-Verbose -Verbose "destination = $destination" Write-Log "replacing $destination with $_" + + if (-not (Test-Path $destination)) { + $parent = Split-Path -Path $destination -Parent + $exists = Test-Path -Path $parent + + if ($exists) { + Write-Verbose -Verbose "Parent:" + Get-ChildItem -Path $parent | Select-Object -ExpandProperty FullName | Write-Verbose -Verbose + } + + Write-Error "File not found: $destination, parent - $parent exists: $exists" + } + + $signature = Get-AuthenticodeSignature -FilePath $_ + + if ($signature.Status -ne 'Valid') { + Write-Error "Invalid signature for $_" + } + Copy-Item -Path $_ -Destination $destination -Force } foreach($filter in $RemoveFilter) { - $removePath = Join-Path -Path $BuildPath -ChildPath $filter + $removePath = Join-Path -Path $BuildPathNormalized -ChildPath $filter Remove-Item -Path $removePath -Recurse -Force } } diff --git a/tools/releaseBuild/setReleaseTag.ps1 b/tools/releaseBuild/setReleaseTag.ps1 index 8d6a8c3eef..60af8c4d74 100644 --- a/tools/releaseBuild/setReleaseTag.ps1 +++ b/tools/releaseBuild/setReleaseTag.ps1 @@ -48,7 +48,14 @@ function New-BuildInfoJson { Write-Verbose -Message "$vstsCommandString" -Verbose Write-Host -Object "##$vstsCommandString" + if (-not (Test-Path $env:ob_outputDirectory)) { + $null = New-Item -Path $env:ob_outputDirectory -ItemType Directory -Force -Verbose + } + + # Upload for ADO pipelines Write-Host "##vso[artifact.upload containerfolder=BuildInfoJson;artifactname=BuildInfoJson]$resolvedPath" + # Copy to location where OneBranch Pipelines uploads from + Copy-Item $resolvedPath -Destination $env:ob_outputDirectory -Force -Verbose } # Script to set the release tag based on the branch name if it is not set or it is "fromBranch"