[release/v7.4] Fixes to Azure Public feed usage (#24429)

This commit is contained in:
Aditya Patwardhan
2024-10-15 15:12:08 -07:00
committed by GitHub
parent 5aac2a7a29
commit 9e9fe79f09
27 changed files with 273 additions and 365 deletions
@@ -58,8 +58,10 @@ variables:
value: ${{ parameters.ReleaseTagVar }}
- name: SKIP_SIGNING
value: ${{ parameters.SKIP_SIGNING }}
- group: 'AzDevOpsArtifacts'
- group: 'mscodehub-feed-read-akv'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: ENABLE_MSBUILD_BINLOGS
value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }}
extends:
template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
@@ -70,6 +72,11 @@ extends:
WindowsHostVersion:
Network: KS2
customTags: 'ES365AIMigrationTooling'
featureFlags:
LinuxHostVersion:
Network: KS3
WindowsHostVersion:
Network: KS3
globalSdl:
disableLegacyManifest: true
# disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates.
@@ -112,15 +119,18 @@ extends:
type: windows
variables:
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson'
- name: ob_sdl_codeSignValidation_enabled
value: false
- name: ob_sdl_codeql_compiled_enabled
value: false
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_signing_setup_enabled
value: false
steps:
- checkout: self
@@ -131,6 +141,8 @@ extends:
- pwsh: |
Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose
displayName: Capture environment variables
env:
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
- template: /.pipelines/templates/SetVersionVariables.yml@self
parameters:
@@ -52,6 +52,7 @@ variables:
value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project
- name: LinuxContainerImage
value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: branchCounterKey
value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])]
+2 -3
View File
@@ -13,9 +13,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
displayName: Delete blob is exists
pool:
@@ -1,39 +1,53 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
- name: "ob_restore_phase"
type: boolean
default: true
steps:
- task: NuGetAuthenticate@1
displayName: Install Azure Artifacts Credential Provider
inputs:
forceReinstallCredentialProvider: true
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
try {
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'"
Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
catch {
Get-Error
throw
}
displayName: 'Switch to production Azure DevOps feed for all nuget.configs'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if (-not (Test-Path $configPath))
{
throw "nuget.config is not created"
Get-ChildItem ${{ parameters.repoRoot }}/nuget.config -Recurse | Foreach-Object {
Write-Verbose -Verbose "--- START $($_.fullname) ---"
get-content $_.fullname | Out-String -width 9999 -Stream | write-Verbose -Verbose
Write-Verbose -Verbose "--- END $($_.fullname) ---"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
displayName: 'Capture all nuget.config files'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}
- pwsh: |
Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose
displayName: Capture VSS* Environment
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
ob_restore_phase: ${{ parameters.ob_restore_phase }}
+9 -9
View File
@@ -20,10 +20,9 @@ jobs:
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
- name: ob_sdl_binskim_enabled
value: true
- name: ob_sdl_credscan_suppressionsfileforartifacts
value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json
- name: PowerShellRoot
value: $(Build.SourcesDirectory)
steps:
- checkout: self
clean: true
@@ -39,19 +38,19 @@ jobs:
# make the current user the owner
sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell"
displayName: 'Create $(Agent.TempDirectory)/PowerShell'
- template: /.pipelines/templates/cloneToOfficialPath.yml@self
parameters:
nativePathRoot: '$(Agent.TempDirectory)'
- pwsh: |
tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
Import-Module $(PowerShellRoot)/build.psm1 -Force
Start-PSBootstrap -Package
displayName: 'Bootstrap VM'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- pwsh: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
$env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
@@ -76,6 +75,7 @@ jobs:
displayName: 'Build'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/step/finalize.yml@self
- job: sign_${{ parameters.buildArchitecture }}
+5 -18
View File
@@ -22,7 +22,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
@@ -89,23 +90,9 @@ jobs:
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- pwsh: |
$repoRoot = "$(PowerShellRoot)"
Write-Verbose -Verbose "repoRoot: $repoRoot"
$configPath = "$repoRoot/nuget.config"
Import-Module "$repoRoot/build.psm1" -Force
New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PowerShellCore_PublicPackages'], ''))
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
@@ -30,9 +30,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
steps:
- checkout: self
@@ -19,7 +19,8 @@ jobs:
- ImageOverride -equals ${{ parameters.imageName }}
variables:
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
+29 -2
View File
@@ -22,15 +22,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@@ -58,6 +65,13 @@ jobs:
BuildTestPackage -runtime win-arm64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task
- job: build_testartifacts_nonwin
variables:
@@ -75,15 +89,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@@ -113,3 +134,9 @@ jobs:
BuildTestPackage -runtime linux-musl-x64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task
+2 -3
View File
@@ -21,9 +21,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
steps:
- checkout: self
+1 -1
View File
@@ -54,7 +54,7 @@ jobs:
displayName: Set Build Name for Non-PR
condition: ne(variables['Build.Reason'], 'PullRequest')
- ${{ if ne(variables['AzDevOpsFeed'], '') }}:
- ${{ if ne(variables['UseAzDevOpsFeed'], '') }}:
- template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
- pwsh: |
+117 -13
View File
@@ -708,6 +708,8 @@ Fix steps:
if ($config.Count -gt 0) {
$configPublishPath = Join-Path -Path $publishPath -ChildPath "powershell.config.json"
Set-Content -Path $configPublishPath -Value ($config | ConvertTo-Json) -Force -ErrorAction Stop
} else {
Write-Warning "No powershell.config.json generated for $publishPath"
}
# Restore the Pester module
@@ -729,25 +731,47 @@ Fix steps:
function Switch-PSNugetConfig {
param(
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[Parameter(Mandatory = $true, ParameterSetName = 'nouser')]
[ValidateSet('Public', 'Private')]
[string] $Source = 'Public'
[string] $Source,
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[string] $UserName,
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[string] $ClearTextPAT
)
Clear-PipelineNugetAuthentication
$extraParams = @()
if ($UserName) {
$extraParams = @{
UserName = $UserName
ClearTextPAT = $ClearTextPAT
}
}
if ( $Source -eq 'Public') {
$dotnetSdk = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v2'; Name = 'dotnet' }
$gallery = [NugetPackageSource] @{Url = 'https://www.powershellgallery.com/api/v2/'; Name = 'psgallery' }
$nugetorg = [NugetPackageSource] @{Url = 'https://api.nuget.org/v3/index.json'; Name = 'nuget.org' }
New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/"
New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/"
New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/" @extraParams
New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/" @extraParams
} elseif ( $Source -eq 'Private') {
$powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json'; Name = 'powershell' }
$powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/PowerShell-7-5-preview-test-2/nuget/v3/index.json'; Name = 'powershell' }
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/"
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/"
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/" @extraParams
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/" @extraParams
} else {
throw "Unknown source: $Source"
}
if ($UserName -or $ClearTextPAT) {
Set-PipelineNugetAuthentication
}
}
function Test-ShouldGenerateExperimentalFeatures
@@ -853,6 +877,10 @@ function Restore-PSPackage
$RestoreArguments += "--interactive"
}
if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
$RestoreArguments += '-bl'
}
$ProjectDirs | ForEach-Object {
$project = $_
Write-Log -message "Run dotnet restore $project $RestoreArguments"
@@ -870,6 +898,23 @@ function Restore-PSPackage
$retryCount++
if($retryCount -ge $maxTries)
{
if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
if ( Test-Path ./msbuild.binlog ) {
if (!(Test-Path $env:OB_OUTPUTDIRECTORY -PathType Container)) {
$null = New-Item -path $env:OB_OUTPUTDIRECTORY -ItemType Directory -Force -Verbose
}
$projectName = Split-Path -Leaf -Path $project
$binlogFileName = "${projectName}.msbuild.binlog"
if ($IsMacOS) {
$resolvedPath = (Resolve-Path -Path ./msbuild.binlog).ProviderPath
Write-Host "##vso[artifact.upload containerfolder=$binLogFileName;artifactname=$binLogFileName]$resolvedPath"
} else {
Copy-Item -Path ./msbuild.binlog -Destination "$env:OB_OUTPUTDIRECTORY/${projectName}.msbuild.binlog" -Verbose
}
}
}
throw
}
continue
@@ -1210,11 +1255,15 @@ function Publish-PSTestTools {
$tools = @(
@{ Path="${PSScriptRoot}/test/tools/TestAlc"; Output="library" }
@{ Path="${PSScriptRoot}/test/tools/TestExe"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/TestService"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/UnixSocket"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/WebListener"; Output="exe" }
)
# This is a windows service, so it only works on windows
if ($environment.IsWindows) {
$tools += @{ Path = "${PSScriptRoot}/test/tools/TestService"; Output = "exe" }
}
$Options = Get-PSOptions -DefaultToNew
# Publish tools so it can be run by tests
@@ -2337,6 +2386,12 @@ function Start-PSBootstrap {
Install-Wix -arm64:$isArm64
}
}
if ($env:TF_BUILD) {
Write-Verbose -Verbose "--- Start - Capturing nuget sources"
dotnet nuget list source --format detailed
Write-Verbose -Verbose "--- End - Capturing nuget sources"
}
} finally {
Pop-Location
}
@@ -3444,6 +3499,8 @@ function New-NugetPackageSource {
return [NugetPackageSource] @{Url = $Url; Name = $Name }
}
$script:NuGetEndpointCredentials = [System.Collections.Generic.Dictionary[String,System.Object]]::new()
function New-NugetConfigFile {
param(
[Parameter(Mandatory = $true, ParameterSetName ='user')]
@@ -3488,22 +3545,53 @@ function New-NugetConfigFile {
'@
$content = $nugetConfigHeaderTemplate
[NugetPackageSource]$source = $null
foreach ($source in $NugetPackageSource) {
$content += $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name)
$feedNamePostfix = ''
if ($UserName) {
$feedNamePostfix += '-' + $UserName.Replace('@', '-').Replace('.', '-')
}
$content += $nugetPackageSourceFooterTemplate
[NugetPackageSource]$source = $null
$newLine = [Environment]::NewLine
foreach ($source in $NugetPackageSource) {
$content += $newLine + $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name + $feedNamePostfix)
}
$content += $newLine + $nugetPackageSourceFooterTemplate
if ($UserName -or $ClearTextPAT) {
$content += $nugetCredentialsTemplate.Replace('[USERNAME]', $UserName).Replace('[PASSWORD]', $ClearTextPAT)
foreach ($source in $NugetPackageSource) {
if (!$script:NuGetEndpointCredentials.ContainsKey($source.Url)) {
$script:NuGetEndpointCredentials.Add($source.Url, @{
endpoint = $source.Url
username = $UserName
password = $ClearTextPAT
})
}
}
}
$content += $nugetConfigFooterTemplate
$content += $newLine + $nugetConfigFooterTemplate
Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force
}
function Clear-PipelineNugetAuthentication {
$script:NuGetEndpointCredentials.Clear()
}
function Set-PipelineNugetAuthentication {
$endpointcredentials = @()
foreach ($key in $script:NuGetEndpointCredentials.Keys) {
$endpointcredentials += $script:NuGetEndpointCredentials[$key]
}
$json = @{
endpointCredentials = $endpointcredentials
} | convertto-json -Compress
Set-PipelineVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $json
}
function Set-CorrectLocale
{
if (-not $IsLinux)
@@ -3637,3 +3725,19 @@ function Update-DotNetSdkVersion {
$dotnetRuntimeMeta.sdk.sdkImageVersion = $version
$dotnetRuntimeMeta | ConvertTo-Json | Out-File $dotnetRuntimeMetaPath
}
function Set-PipelineVariable {
param(
[parameter(Mandatory)]
[string] $Name,
[parameter(Mandatory)]
[string] $Value
)
$vstsCommandString = "vso[task.setvariable variable=$Name]$Value"
Write-Verbose -Verbose -Message ("sending " + $vstsCommandString)
Write-Host "##$vstsCommandString"
# also set in the current session
Set-Item -Path "env:$Name" -Value $Value
}
+1 -1
View File
@@ -2,7 +2,7 @@
<configuration>
<packageSources>
<clear />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json" />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell-dotnet-9/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
@@ -32,13 +32,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />
<PackageReference Include="System.Threading.AccessControl" Version="8.0.0" />
<PackageReference Include="System.Drawing.Common" Version="8.0.10" />
<PackageReference Include="JsonSchema.Net" Version="5.2.7" />
<!-- This section is to force the version of non-direct dependencies -->
<PackageReference Include="Json.More.Net" Version="1.9.3" />
<PackageReference Include="System.Text.Json" Version="8.0.5" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
<PackageReference Include="System.Threading.AccessControl" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="System.Drawing.Common" Version="9.0.0-preview.6.24327.6" />
<PackageReference Include="JsonSchema.Net" Version="7.0.4" />
</ItemGroup>
</Project>
+1 -1
View File
@@ -2,7 +2,7 @@
<configuration>
<packageSources>
<clear />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json" />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell-dotnet-9/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
+5 -4
View File
@@ -10,13 +10,14 @@
<TieredCompilation>true</TieredCompilation>
<TieredCompilationQuickJit>true</TieredCompilationQuickJit>
<RuntimeIdentifiers>win-x86;win-x64</RuntimeIdentifiers>
<TargetPlatformIdentifier>Windows</TargetPlatformIdentifier>
<TargetPlatformVersion>8.0</TargetPlatformVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Windows.Compatibility" Version="8.0.1" />
<!-- the following dependencies are transitive -->
<PackageReference Include="System.IO.Packaging" Version="8.0.1" />
<PackageReference Include="System.Runtime.Caching" Version="8.0.1" />
<PackageReference Include="Microsoft.Windows.Compatibility" Version="9.0.0-preview.6.24327.6" />
<PackageReference Include="System.Formats.Asn1" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
</ItemGroup>
</Project>
+1 -6
View File
@@ -7,12 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.10" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="2.1.0-*" />
</ItemGroup>
</Project>
+8 -12
View File
@@ -395,6 +395,10 @@
"Pattern": "hostpolicy.dll",
"FileType": "NonProduct"
},
{
"Pattern": "Humanizer.dll",
"FileType": "NonProduct"
},
{
"Pattern": "it/Microsoft.CodeAnalysis.CSharp.resources.dll",
"FileType": "NonProduct"
@@ -563,10 +567,6 @@
"Pattern": "ja/WindowsFormsIntegration.resources.dll",
"FileType": "NonProduct"
},
{
"Pattern": "JetBrains.Annotations.dll",
"FileType": "NonProduct"
},
{
"Pattern": "Json.More.dll",
"FileType": "NonProduct"
@@ -667,10 +667,6 @@
"Pattern": "Markdig.Signed.dll",
"FileType": "NonProduct"
},
{
"Pattern": "mi.dll",
"FileType": "NonProduct"
},
{
"Pattern": "Microsoft.ApplicationInsights.dll",
"FileType": "NonProduct"
@@ -787,10 +783,6 @@
"Pattern": "Microsoft.WSMan.Runtime.xml",
"FileType": "NonProduct"
},
{
"Pattern": "miutils.dll",
"FileType": "NonProduct"
},
{
"Pattern": "Modules/*.json",
"FileType": "NonProduct"
@@ -2599,6 +2591,10 @@
"Pattern": "System.Private.Uri.dll",
"FileType": "NonProduct"
},
{
"Pattern": "System.Private.Windows.Core.dll",
"FileType": "NonProduct"
},
{
"Pattern": "System.Private.Xml.dll",
"FileType": "NonProduct"
@@ -19,12 +19,6 @@ steps:
displayName: Update PSGet and PackageManagement
condition: succeededOrFailed()
- pwsh: |
Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1"
SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory)
displayName: Download packages from PSGallery that need to be updated
condition: succeededOrFailed()
- pwsh: |
Write-Verbose -Verbose "Packages to upload"
if(Test-Path $(Build.ArtifactStagingDirectory)) { Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg" | ForEach-Object { $_.FullName }}
@@ -1,186 +0,0 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
<#
.SYNOPSIS
Downloads to packages from PowerShell Gallery which are missing from the Azure DevOps Artifacts feed.
.PARAMETER AzureDevOpsPAT
PAT for the username used for authenticating to the Azure DevOps Artifacts feed.
.PARAMETER Destination
Path to the folder where the packages should be stored for uploading to Azure DevOps Artifacts feed.
#>
function SyncGalleryToAzArtifacts {
param(
[Parameter(Mandatory = $true)] [string] $AzDevOpsFeedUserName,
[Parameter(Mandatory = $true)] [string] $AzDevOpsPAT,
[Parameter(Mandatory = $true)] [string] $Destination
)
$csproj = [xml] (Get-Content 'src/Modules/PSGalleryModules.csproj')
$packages = @($csproj.Project.ItemGroup.PackageReference | ForEach-Object { [ordered] @{Name = $_.Include; Version = $_.Version }})
$galleryPackages = @()
$azArtifactsPackages = @()
$modulesToUpdate = @()
$galleryUrl = 'https://www.powershellgallery.com/api/v2/'
$azArtifactsUrl = 'https://mscodehub.pkgs.visualstudio.com/_packaging/pscore-release/nuget/v2'
$azDevOpsCreds = [pscredential]::new($AzDevOpsFeedUserName, (ConvertTo-SecureString -String $AzDevOpsPAT -AsPlainText -Force))
foreach ($package in $packages) {
try {
# Get module from gallery
$foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $package.Name -Force -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnGallery.Version) in gallery"
$galleryPackages += $foundPackageOnGallery
} catch {
if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
# Log and ignore failure is required version is not found on gallery.
Write-Warning "Module not found on gallery $($package.Name) - $($package.Version)"
}
else {
Write-Error $_
}
}
try {
# Get module from Az Artifacts
# There seems to be a bug in the feed with RequiredVersion matching. Adding workaround with post filtering.
# Issue: https://github.com/OneGet/oneget/issues/397
$foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $package.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnAz.Version) in azArtifacts"
$azArtifactsPackages += $foundPackageOnAz
} catch {
if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
# Log and add the module to update list.
Write-Verbose -Verbose "Az Artifacts Module needs update to - $($package.Name) - $($package.Version)"
$modulesToUpdate += $package
}
else {
Write-Error $_
}
}
# Check if Az package version is less that gallery version
$pkgOnAzVersion = [semver]::new($foundPackageOnAz.Version)
$pkgOnGalleryVersion = [semver]::new($foundPackageOnGallery.Version)
if ($pkgOnAzVersion -lt $pkgOnGalleryVersion) {
Write-Verbose -Verbose "Module needs to be updated $($package.Name) - $($foundPackageOnGallery.Version)"
$modulesToUpdate += $foundPackageOnGallery
} elseif ($pkgOnGalleryVersion -lt $pkgOnAzVersion) {
Write-Warning "Newer version found on Az Artifacts - $($foundPackageOnAz.Name) - $($foundPackageOnAz.Version)"
} else {
Write-Verbose -Verbose "Module is in sync - $($package.Name)"
}
}
"`nGallery Packages:"
$galleryPackages
"`nAz Artifacts Packages:`n"
$azArtifactsPackages
"`nModules to update:`n"
$modulesToUpdate
foreach ($package in $modulesToUpdate) {
Write-Verbose -Verbose "Saving package $($package.Name) - $($package.Version)"
Save-Package -Provider NuGet -Source $galleryUrl -Name $package.Name -RequiredVersion $package.Version -Path $Destination
}
if ($modulesToUpdate.Length -gt 0)
{
# Remove dependent packages downloaded by Save-Package if there are already present in AzArtifacts feed.
try {
$null = Register-PackageSource -Name local -Location $Destination -ProviderName NuGet -Force
$packageNamesToKeep = @()
$savedPackages = Find-Package -Source local -AllVersions -AllowPreReleaseVersion
Write-Verbose -Verbose "Saved packages:"
$savedPackages | Out-String | Write-Verbose -Verbose
foreach($package in $savedPackages) {
$pkgVersion = NormalizeVersion -version $package.Version
$foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $package.Name -and (NormalizeVersion -version $_.Version) -eq $pkgVersion }
if(-not $foundMatch) {
Write-Verbose "Keeping package $($package.PackageFileName)" -Verbose
$packageNamesToKeep += "{0}*.nupkg" -f $package.Name
}
}
if ($packageNamesToKeep.Length -gt 0) {
## Removing only if we do have some packages to keep,
## otherwise the '$Destination' folder will be removed.
Remove-Item -Path $Destination -Exclude $packageNamesToKeep -Recurse -Force -Verbose
}
Write-Verbose -Verbose "Packages kept for upload"
Get-ChildItem $Destination | Out-String | Write-Verbose -Verbose
}
finally {
Unregister-PackageSource -Name local -Force -ErrorAction SilentlyContinue
}
}
}
Function SortPackage {
param(
[Parameter(ValueFromPipeline = $true)]
[Microsoft.PackageManagement.Packaging.SoftwareIdentity[]]
$packages
)
Begin {
$allPackages = @()
}
Process {
$allPackages += $packages
}
End {
$versions = $allPackages.Version |
ForEach-Object { ($_ -split '-')[0] } |
Select-Object -Unique |
Sort-Object -Descending -Property Version
foreach ($version in $versions) {
$exactMatch = $allPackages | Where-Object {
Write-Verbose "testing $($_.version) -eq $version"
$_.version -eq $version
}
if ($exactMatch) {
Write-Output $exactMatch
}
$allPackages | Where-Object {
$_.version -like "${version}-*"
} | Sort-Object -Descending -Property Version | Write-Output
}
}
}
function NormalizeVersion {
param ([string] $version)
$sVer = if ($version -match "(\d+.\d+.\d+).0") {
$Matches[1]
} elseif ($version -match "^\d+.\d+$") {
# Two digit versions are stored as three digit versions
"$version.0"
} else {
$version
}
$sVer
}
Export-ModuleMember -Function 'SyncGalleryToAzArtifacts', 'SortPackage'
@@ -362,7 +362,8 @@ stages:
variables:
- group: 'Azure Blob variable group'
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: ReleasePipelineSecrets
steps:
- template: templates/release-CreateGitHubDraft.yml
@@ -404,9 +405,10 @@ stages:
- ImageOverride -equals PSMMSUbuntu20.04-Secure
variables:
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: 'packages.microsoft.com'
- group: 'mscodehub-feed-read-akv'
- group: 'mscodehub-code-read-akv'
steps:
- template: templates/release-PublishPackageMsftCom.yml
parameters:
@@ -19,7 +19,8 @@ jobs:
- group: DotNetPrivateBuildAccess
- group: Azure Blob variable group
- group: ReleasePipelineSecrets
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
pool:
name: PowerShell1ES
@@ -1,38 +1,8 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
- name: "repoRoot"
default: $(REPOROOT)
steps:
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(REPOROOT)
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}
- task: nuget-security-analysis@0
displayName: 'Run Secure Supply Chain analysis'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''), ne(variables['AzDevOpsFeed'], ''))
@@ -50,7 +50,7 @@ jobs:
repoRoot: $(PowerShellRoot)
- pwsh: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
$env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
$(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks
@@ -20,18 +20,9 @@ parameters:
steps:
- pwsh: |
$configPath = "$(REPOROOT)/nuget.config"
Import-Module '$(REPOROOT)/build.psm1' -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "$(REPOROOT)"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(REPOROOT)
- pwsh: |
Import-Module "$env:REPOROOT/build.psm1" -Force
@@ -19,7 +19,7 @@ steps:
"--verbose",
"--branch",
"$branch",
"https://$(mscodehubPackageReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
"https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
'$(Pipeline.Workspace)/tools'
$gitArgs | Write-Verbose -Verbose
git $gitArgs
@@ -10,7 +10,8 @@ jobs:
# testing
vmImage: ${{ parameters.imageName }}
variables:
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
- checkout: self
@@ -39,9 +40,12 @@ jobs:
artifact: metadata
path: '$(Pipeline.Workspace)/releasePipeline/metadata'
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
- pwsh: |
Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination '$(Build.SourcesDirectory)/test/hosting'
Write-Verbose -Verbose "Capture hosting folder files"
Get-ChildItem '$(Build.SourcesDirectory)/test/hosting'