[release/v7.4] Split TPN manifest and Component Governance manifest (#26891) (#26961)

Co-authored-by: Travis Plunk <travis.plunk@microsoft.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Justin Chung
2026-03-09 16:23:19 -05:00
committed by GitHub
co-authored by Travis Plunk Copilot
parent db2ae08534
commit 7d39838fa5
12 changed files with 1411 additions and 23 deletions
@@ -55,7 +55,7 @@ jobs:
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
inputs:
sourceScanPath: '$(repoRoot)\tools'
sourceScanPath: '$(repoRoot)\tools\cgmanifest\tpn'
- pwsh: |
$(repoRoot)/tools/clearlyDefined/ClearlyDefined.ps1 -TestAndHarvest
+1 -1
View File
@@ -34,7 +34,7 @@ pr:
- .vsts-ci/misc-analysis.yml
- .vsts-ci/windows.yml
- .vsts-ci/windows/*
- tools/cgmanifest.json
- tools/cgmanifest/*
- LICENSE.txt
- test/common/markdown/*
- test/perf/*
+1 -1
View File
@@ -32,7 +32,7 @@ pr:
- .vsts-ci/misc-analysis.yml
- .vsts-ci/windows.yml
- .vsts-ci/windows/*
- tools/cgmanifest.json
- tools/cgmanifest/*
- LICENSE.txt
- test/common/markdown/*
- test/perf/*
+1 -1
View File
@@ -34,7 +34,7 @@ pr:
- .github/ISSUE_TEMPLATE/*
- .github/workflows/*
- .vsts-ci/misc-analysis.yml
- tools/cgmanifest.json
- tools/cgmanifest/*
- LICENSE.txt
- test/common/markdown/*
- test/perf/*
+1 -1
View File
@@ -28,7 +28,7 @@ pr:
- .dependabot/config.yml
- .github/ISSUE_TEMPLATE/*
- .vsts-ci/misc-analysis.yml
- tools/cgmanifest.json
- tools/cgmanifest/*
- LICENSE.txt
- test/common/markdown/*
- test/perf/*
+755
View File
@@ -0,0 +1,755 @@
{
"Registrations": [
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "DotNetAnalyzers.DocumentationAnalyzers.Unstable",
"Version": "1.0.0.59"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "DotNetAnalyzers.DocumentationAnalyzers",
"Version": "1.0.0-beta.59"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Humanizer.Core",
"Version": "2.14.1"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Json.More.Net",
"Version": "2.1.1"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "JsonPointer.Net",
"Version": "5.3.1"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "JsonSchema.Net",
"Version": "7.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Markdig.Signed",
"Version": "0.45.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.ApplicationInsights",
"Version": "2.23.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Bcl.AsyncInterfaces",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.CodeAnalysis.Analyzers",
"Version": "3.11.0"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.CodeAnalysis.Common",
"Version": "5.0.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.CodeAnalysis.CSharp",
"Version": "5.0.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Extensions.ObjectPool",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Management.Infrastructure.Runtime.Win",
"Version": "3.0.0"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.PowerShell.MarkdownRender",
"Version": "7.2.1"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Security.Extensions",
"Version": "1.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Win32.Registry.AccessControl",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Win32.SystemEvents",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Microsoft.Windows.Compatibility",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "Newtonsoft.Json",
"Version": "13.0.4"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.android-arm.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.android-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.android-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.android-x86.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-arm.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-bionic-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-bionic-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-musl-arm.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-musl-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-musl-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.linux-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.maccatalyst-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.maccatalyst-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.native.System.Data.SqlClient.sni",
"Version": "4.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.osx-arm64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.osx-x64.runtime.native.System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.win-arm64.runtime.native.System.Data.SqlClient.sni",
"Version": "4.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.win-x64.runtime.native.System.Data.SqlClient.sni",
"Version": "4.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "runtime.win-x86.runtime.native.System.Data.SqlClient.sni",
"Version": "4.4.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "StyleCop.Analyzers.Unstable",
"Version": "1.2.0.556"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "StyleCop.Analyzers",
"Version": "1.1.118"
}
},
"DevelopmentDependency": true
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.CodeDom",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ComponentModel.Composition.Registration",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ComponentModel.Composition",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Configuration.ConfigurationManager",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Data.Odbc",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Data.OleDb",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Data.SqlClient",
"Version": "4.9.0"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Diagnostics.EventLog",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Diagnostics.PerformanceCounter",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.DirectoryServices.AccountManagement",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.DirectoryServices.Protocols",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.DirectoryServices",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Drawing.Common",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.IO.Packaging",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.IO.Ports",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Management",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Net.Http.WinHttpHandler",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Reflection.Context",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Runtime.Caching",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Security.Cryptography.Pkcs",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Security.Cryptography.ProtectedData",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Security.Cryptography.Xml",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Security.Permissions",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceModel.Http",
"Version": "10.0.652802"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceModel.NetFramingBase",
"Version": "10.0.652802"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceModel.NetTcp",
"Version": "10.0.652802"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceModel.Primitives",
"Version": "10.0.652802"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceModel.Syndication",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.ServiceProcess.ServiceController",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Speech",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Web.Services.Description",
"Version": "8.1.2"
}
},
"DevelopmentDependency": false
},
{
"Component": {
"Type": "nuget",
"Nuget": {
"Name": "System.Windows.Extensions",
"Version": "10.0.3"
}
},
"DevelopmentDependency": false
}
],
"$schema": "https://json.schemastore.org/component-detection-manifest.json"
}
+1 -1
View File
@@ -21,7 +21,7 @@ if ($ForceModuleReload) {
Import-Module -Name "$PSScriptRoot/src/ClearlyDefined" @extraParams
$cgManfest = Get-Content "$PSScriptRoot/../cgmanifest.json" | ConvertFrom-Json
$cgManfest = Get-Content "$PSScriptRoot/../cgmanifest/main/cgmanifest.json" | ConvertFrom-Json
$fullCgList = $cgManfest.Registrations.Component |
ForEach-Object {
[Pscustomobject]@{
@@ -0,0 +1,156 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
<#
.SYNOPSIS
Find the last harvested version of a NuGet package from ClearlyDefined.
.DESCRIPTION
Searches for the last harvested version of a package by checking versions
backwards from the specified current version. This is useful for reverting
to a known-good harvested version when a newer version hasn't been harvested yet.
.PARAMETER Name
The NuGet package name to search for.
.PARAMETER CurrentVersion
The version to start searching backwards from. Version comparison uses semantic versioning.
.PARAMETER PackageSourceName
The NuGet package source name to use when searching for available versions.
Default is 'findMissingNoticesNugetOrg' if not specified.
.EXAMPLE
Find-LastHarvestedVersion -Name "Microsoft.Windows.Compatibility" -CurrentVersion "8.0.24"
# This will return "8.0.22" if that's the last harvested version
.NOTES
Requires the ClearlyDefined module to be imported:
Import-Module ".\clearlyDefined\src\ClearlyDefined" -Force
#>
function Find-LastHarvestedVersion {
[CmdletBinding()]
param(
[parameter(Mandatory)]
[string]$Name,
[parameter(Mandatory)]
[string]$CurrentVersion,
[string]$PackageSourceName = 'findMissingNoticesNugetOrg'
)
try {
Write-Verbose "Finding last harvested version for $Name starting from v$CurrentVersion..."
# Parse the current version
try {
[System.Management.Automation.SemanticVersion]$currentSemVer = $CurrentVersion
} catch {
[Version]$currentSemVer = $CurrentVersion
}
# First try the ClearlyDefined search API (more efficient)
try {
Write-Verbose "Searching ClearlyDefined API for versions of $Name (sorted by release date)..."
# Get versions sorted by release date descending (newest first) for efficiency
$versions = Get-ClearlyDefinedPackageVersions -PackageName $Name
if ($versions -and $versions.Count -gt 0) {
# Results are already sorted by release date newest first
# Filter to versions <= current version
foreach ($versionInfo in $versions) {
try {
$versionObj = [System.Management.Automation.SemanticVersion]$versionInfo.Version
if ($versionObj -le $currentSemVer) {
# Check harvest status
if ($versionInfo.Harvested) {
Write-Verbose "Found harvested version: v$($versionInfo.Version)"
return $versionInfo.Version
} else {
Write-Verbose "v$($versionInfo.Version) - Not harvested, continuing..."
}
}
} catch {
# Skip versions that can't be parsed
}
}
Write-Verbose "No harvested version found in ClearlyDefined results"
return $null
}
} catch {
Write-Verbose "ClearlyDefined search API failed ($_), falling back to NuGet search..."
}
# Fallback: Get all available versions from NuGet and check individually
Write-Verbose "Falling back to NuGet source search..."
# Ensure package source exists
if (!(Get-PackageSource -Name $PackageSourceName -ErrorAction SilentlyContinue)) {
Write-Verbose "Registering package source: $PackageSourceName"
$null = Register-PackageSource -Name $PackageSourceName -Location https://www.nuget.org/api/v2 -ProviderName NuGet
}
# Get all available versions from NuGet
try {
$allVersions = Find-Package -Name $Name -AllowPrereleaseVersions -source $PackageSourceName -AllVersions -ErrorAction SilentlyContinue | ForEach-Object {
try {
$packageVersion = [System.Management.Automation.SemanticVersion]$_.Version
} catch {
$packageVersion = [Version]$_.Version
}
$_ | Add-Member -Name SemVer -MemberType NoteProperty -Value $packageVersion -PassThru
} | Where-Object { $_.SemVer -le $currentSemVer } | Sort-Object -Property SemVer -Descending | ForEach-Object { $_.Version }
} catch {
Write-Warning "Failed to get versions for $Name : $_"
return $null
}
if (!$allVersions) {
Write-Verbose "No versions found for $Name"
return $null
}
# Check each version backwards until we find one that's harvested
foreach ($version in $allVersions) {
$pkg = [PSCustomObject]@{
type = "nuget"
Name = $Name
PackageVersion = $version
}
try {
$result = $pkg | Get-ClearlyDefinedData
if ($result -and $result.harvested) {
Write-Verbose "Found harvested version: v$version"
return $version
} else {
Write-Verbose "v$version - Not harvested, continuing..."
}
} catch {
Write-Verbose "Error checking v$version : $_" -Verbose
}
}
Write-Verbose "No harvested version found for $Name"
return $null
} finally {
Save-ClearlyDefinedCache
}
}
# If this script is called directly (not sourced), run a test
if ($MyInvocation.InvocationName -eq '.' -or $MyInvocation.Line -like '. "*Find-LastHarvestedVersion*') {
# Script was sourced, just load the function
} else {
# Script was called directly
Write-Host "Testing Find-LastHarvestedVersion function..."
Write-Host "Ensure ClearlyDefined module is loaded first:"
Write-Host ' Import-Module ".\clearlydefined\src\ClearlyDefined" -Force'
Write-Host ""
Write-Host "Example usage:"
Write-Host ' Find-LastHarvestedVersion -Name "Microsoft.Windows.Compatibility" -CurrentVersion "8.0.24"'
}
@@ -2,6 +2,9 @@
# Licensed under the MIT License.
# Start the collection (known as harvest) of ClearlyDefined data for a package
$retryIntervalSec = 90
$maxRetryCount = 5
function Start-ClearlyDefinedHarvest {
[CmdletBinding()]
param(
@@ -27,7 +30,9 @@ function Start-ClearlyDefinedHarvest {
$coordinates = Get-ClearlyDefinedCoordinates @PSBoundParameters
$body = @{tool='package';coordinates=$coordinates} | convertto-json
Write-Verbose $body -Verbose
(Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json').Content
Start-job -ScriptBlock {
Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $using:body -ContentType 'application/json' -MaximumRetryCount $using:maxRetryCount -RetryIntervalSec $using:retryIntervalSec
}
}
}
@@ -74,6 +79,207 @@ Function Get-ClearlyDefinedCoordinates {
# Cache of ClearlyDefined data
$cdCache = @{}
function Test-ClearlyDefinedCachePersistenceAllowed {
[CmdletBinding()]
param()
if ($env:TF_BUILD -or $env:ADO_BUILD_ID -or $env:BUILD_BUILDID) {
return $false
}
if ($env:GITHUB_ACTIONS -or $env:GITHUB_RUN_ID) {
return $false
}
return $true
}
function Get-ClearlyDefinedCachePath {
[CmdletBinding()]
param()
$tempPath = [System.IO.Path]::GetTempPath()
return (Join-Path -Path $tempPath -ChildPath 'clearlydefined-cache.json')
}
function Save-ClearlyDefinedCache {
[CmdletBinding()]
param()
if (-not (Test-ClearlyDefinedCachePersistenceAllowed)) {
Write-Verbose 'Skipping cache persistence for CI environment.'
return
}
if ($cdCache.Count -eq 0) {
Write-Verbose 'No cache entries to persist.'
return
}
$cachePath = Get-ClearlyDefinedCachePath
$entries = foreach ($key in $cdCache.Keys) {
[PSCustomObject]@{
coordinates = $key
data = $cdCache[$key]
}
}
$cachePayload = @{
savedAtUtc = (Get-Date).ToUniversalTime()
entries = $entries
} | ConvertTo-Json -Depth 20
$cachePayload | Set-Content -Path $cachePath -Encoding UTF8
Write-Verbose "Persisted cache to $cachePath"
}
function Import-ClearlyDefinedCache {
[CmdletBinding()]
param()
if (-not (Test-ClearlyDefinedCachePersistenceAllowed)) {
Write-Verbose 'Skipping cache import for CI environment.'
return
}
$cachePath = Get-ClearlyDefinedCachePath
if (-not (Test-Path -Path $cachePath)) {
Write-Verbose 'No persisted cache found.'
return
}
try {
$payload = Get-Content -Path $cachePath -Raw | ConvertFrom-Json
} catch {
Write-Verbose "Failed to read cache file: $cachePath"
return
}
if (-not $payload.entries) {
Write-Verbose 'Cache file did not contain entries.'
return
}
foreach ($entry in $payload.entries) {
if (-not $entry.coordinates -or -not $entry.data) {
continue
}
try {
$entry.data.cachedTime = [datetime]$entry.data.cachedTime
} catch {
continue
}
$cdCache[$entry.coordinates] = $entry.data
}
Write-Verbose "Imported $($cdCache.Count) cache entries from $cachePath"
}
# Search for packages in ClearlyDefined
Function Search-ClearlyDefined {
[CmdletBinding()]
param(
[string]$Type = 'nuget',
[string]$Provider = 'nuget',
[string]$Namespace,
[string]$Name,
[string]$Pattern,
[datetime]$ReleasedAfter,
[datetime]$ReleasedBefore,
[ValidateSet('releaseDate', 'name')]
[string]$Sort,
[switch]$SortDesc
)
$queryParams = @()
if ($Type) { $queryParams += "type=$([System.Uri]::EscapeDataString($Type))" }
if ($Provider) { $queryParams += "provider=$([System.Uri]::EscapeDataString($Provider))" }
if ($Namespace) { $queryParams += "namespace=$([System.Uri]::EscapeDataString($Namespace))" }
if ($Name) { $queryParams += "name=$([System.Uri]::EscapeDataString($Name))" }
if ($Pattern) { $queryParams += "pattern=$([System.Uri]::EscapeDataString($Pattern))" }
if ($ReleasedAfter) { $queryParams += "releasedAfter=$($ReleasedAfter.ToString('o'))" }
if ($ReleasedBefore) { $queryParams += "releasedBefore=$($ReleasedBefore.ToString('o'))" }
if ($Sort) { $queryParams += "sort=$([System.Uri]::EscapeDataString($Sort))" }
if ($SortDesc) { $queryParams += "sortDesc=true" }
$searchUri = "https://api.clearlydefined.io/definitions?" + ($queryParams -join '&')
Write-Verbose "Searching ClearlyDefined: $searchUri"
try {
$results = Invoke-RestMethod -Uri $searchUri -MaximumRetryCount $maxRetryCount -RetryIntervalSec $retryIntervalSec
return $results
} catch {
if ($retryIntervalSec -lt 300) {
$retryIntervalSec++
}
Write-Warning "Failed to search ClearlyDefined: $_"
return $null
}
}
# Get available versions for a NuGet package with harvest status
Function Get-ClearlyDefinedPackageVersions {
[CmdletBinding()]
param(
[parameter(mandatory = $true)]
[string]
$PackageName,
[validateset('nuget')]
[string]
$PackageType = 'nuget'
)
# Search for all definitions of this package, sorted by release date (newest first)
Write-Verbose "Fetching versions of $PackageName from ClearlyDefined..."
$results = Search-ClearlyDefined -Type $PackageType -Provider nuget -Name $PackageName -Sort releaseDate -SortDesc
if (!$results) {
Write-Verbose "No results found for $PackageName"
return @()
}
# Convert results to version info objects
$versions = @()
# API returns results in different formats depending on the query
$dataArray = $null
if ($results.data) {
$dataArray = $results.data
} elseif ($results -is [array]) {
$dataArray = $results
} elseif ($results.PSObject.Properties.Count -gt 0) {
# If it's an object with properties, try to extract the actual results
foreach ($prop in $results.PSObject.Properties) {
if ($prop.Value -is [object] -and $prop.Value.revision) {
$dataArray += $prop.Value
}
}
}
if ($dataArray) {
foreach ($item in $dataArray) {
if ($item.revision) {
$harvested = if ($item.licensed -and $item.licensed.declared) { $true } else { $false }
$versions += [PSCustomObject]@{
Name = $item.name
Version = $item.revision
Harvested = $harvested
Licensed = $item.licensed.declared
}
}
}
}
# Results are already sorted by API, no need to re-sort
return $versions
}
# Get the ClearlyDefined data for a package
Function Get-ClearlyDefinedData {
[CmdletBinding()]
@@ -96,8 +302,9 @@ Function Get-ClearlyDefinedData {
)
Begin {
$cacheMinutes = 60
$cacheCutoff = (get-date).AddMinutes(-$cacheMinutes)
# Different TTLs for different cache types
$harvestedCacheMinutes = 60 # Cache positive results for 60 minutes
$nonHarvestedCacheMinutes = 30 # Cache negative results for 30 minutes (less aggressive)
$coordinateList = @()
}
@@ -111,19 +318,55 @@ Function Get-ClearlyDefinedData {
foreach($coordinates in $coordinateList) {
Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates" -PercentComplete (($completed / $total) * 100)
$containsKey = $cdCache.ContainsKey($coordinates)
if ($containsKey -and $cdCache[$coordinates].cachedTime -gt $cacheCutoff) {
Write-Verbose "Returning cached data for $coordinates"
Write-Output $cdCache[$coordinates]
continue
if ($containsKey) {
$cached = $cdCache[$coordinates]
# Check if cache entry is still valid based on its type
$cacheCutoff = if ($cached.harvestedResult) {
(get-date).AddMinutes(-$harvestedCacheMinutes)
} else {
(get-date).AddMinutes(-$nonHarvestedCacheMinutes)
}
if ($cached.cachedTime -gt $cacheCutoff) {
Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates - cache hit" -PercentComplete (($completed / $total) * 100)
Write-Verbose "Returning cached data for $coordinates (harvested: $($cached.harvestedResult))"
Write-Output $cached
$completed++
continue
}
}
Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" | ForEach-Object {
[bool] $harvested = if ($_.licensed.declared) { $true } else { $false }
Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru | Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru
if ($_.harvested) {
Write-Verbose "Caching data for $coordinates"
$cdCache[$coordinates] = $_
Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates - cache miss" -PercentComplete (($completed / $total) * 100)
try {
Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" -MaximumRetryCount $maxRetryCount -RetryIntervalSec $retryIntervalSec | ForEach-Object {
[bool] $harvested = if ($_.licensed.declared) { $true } else { $false }
# Always cache, with harvestedResult property to distinguish for TTL purposes
Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru |
Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru |
Add-Member -NotePropertyName harvestedResult -NotePropertyValue $harvested -PassThru |
ForEach-Object {
Write-Verbose "Caching data for $coordinates (harvested: $($_.harvested))"
$cdCache[$coordinates] = $_
Write-Output $_
}
}
} catch {
if ($retryIntervalSec -lt 300) {
$retryIntervalSec++
}
Write-Warning "Failed to get ClearlyDefined data for $coordinates : $_"
# Return a minimal object indicating failure/not harvested so the pipeline continues
$failedResult = [PSCustomObject]@{
coordinates = $coordinates
harvested = $false
harvestedResult = $false
cachedTime = (get-date)
licensed = @{ declared = $null }
}
Write-Output $failedResult
}
$completed++
}
@@ -134,4 +377,10 @@ Export-ModuleMember -Function @(
'Start-ClearlyDefinedHarvest'
'Get-ClearlyDefinedData'
'ConvertFrom-ClearlyDefinedCoordinates'
'Search-ClearlyDefined'
'Get-ClearlyDefinedPackageVersions'
'Save-ClearlyDefinedCache'
'Import-ClearlyDefinedCache'
'Test-ClearlyDefinedCachePersistenceAllowed'
'Get-ClearlyDefinedCachePath'
)
+231 -3
View File
@@ -7,12 +7,14 @@
param(
[switch] $Fix,
[switch] $IsStable
[switch] $IsStable,
[switch] $ForceHarvestedOnly
)
Import-Module dotnet.project.assets
Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force
. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1"
. "$PSScriptRoot\clearlyDefined\Find-LastHarvestedVersion.ps1"
$packageSourceName = 'findMissingNoticesNugetOrg'
if (!(Get-PackageSource -Name $packageSourceName -ErrorAction SilentlyContinue)) {
@@ -20,7 +22,7 @@ if (!(Get-PackageSource -Name $packageSourceName -ErrorAction SilentlyContinue))
}
$existingRegistrationTable = @{}
$cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\tools\cgmanifest.json).ProviderPath
$cgManifestPath = (Resolve-Path -Path $PSScriptRoot\cgmanifest\main\cgmanifest.json).ProviderPath
$existingRegistrationsJson = Get-Content $cgManifestPath | ConvertFrom-Json -AsHashtable
$existingRegistrationsJson.Registrations | ForEach-Object {
$registration = [Registration]$_
@@ -343,8 +345,91 @@ if ($IsStable) {
}
$count = $newRegistrations.Count
$registrationsToSave = $newRegistrations
$tpnRegistrationsToSave = $null
# If -ForceHarvestedOnly is specified with -Fix, only include harvested packages
# and revert non-harvested packages to their previous versions
if ($Fix -and $ForceHarvestedOnly) {
Write-Verbose "Checking harvest status and filtering to harvested packages with reversion..." -Verbose
# Import ClearlyDefined module to check harvest status
Import-Module -Name "$PSScriptRoot/clearlyDefined/src/ClearlyDefined" -Force
# Import cache from previous runs to speed up lookups
Import-ClearlyDefinedCache
# Get harvest data for all registrations
$fullCgList = $newRegistrations |
ForEach-Object {
[PSCustomObject]@{
type = $_.Component.Type
Name = $_.Component.Nuget.Name
PackageVersion = $_.Component.Nuget.Version
}
}
$fullList = $fullCgList | Get-ClearlyDefinedData
# Build a lookup table of harvest status by package name + version
$harvestStatus = @{}
foreach ($item in $fullList) {
$key = "$($item.Name)|$($item.PackageVersion)"
$harvestStatus[$key] = $item.harvested
}
# Build a lookup table of old versions from existing manifest
$oldVersions = @{}
foreach ($registration in $existingRegistrationsJson.Registrations) {
$name = $registration.Component.Nuget.Name
if (!$oldVersions.ContainsKey($name)) {
$oldVersions[$name] = $registration
}
}
# Process each new registration: keep harvested, revert non-harvested
$tpnRegistrationsToSave = @()
$harvestedCount = 0
$revertedCount = 0
foreach ($reg in $newRegistrations) {
$name = $reg.Component.Nuget.Name
$version = $reg.Component.Nuget.Version
$key = "$name|$version"
if ($harvestStatus.ContainsKey($key) -and $harvestStatus[$key]) {
# Package is harvested, include it
$tpnRegistrationsToSave += $reg
$harvestedCount++
} else {
# Package not harvested, find last harvested version
$lastHarvestedVersion = Find-LastHarvestedVersion -Name $name -CurrentVersion $version
# Use last harvested version if found, otherwise use old version as fallback
if ($lastHarvestedVersion) {
if ($lastHarvestedVersion -ne $version) {
$revertedReg = New-NugetComponent -Name $name -Version $lastHarvestedVersion -DevelopmentDependency:$reg.DevelopmentDependency
$tpnRegistrationsToSave += $revertedReg
$revertedCount++
Write-Verbose "Reverted $name from v$version to last harvested v$lastHarvestedVersion" -Verbose
} else {
$tpnRegistrationsToSave += $reg
}
} elseif ($oldVersions.ContainsKey($name)) {
$tpnRegistrationsToSave += $oldVersions[$name]
$revertedCount++
Write-Verbose "Reverted $name to previous version (no harvested version found)" -Verbose
} else {
Write-Warning "$name v$version not harvested and no previous version found. Excluding from manifest."
}
}
}
Write-Verbose "Completed filtering for TPN: $harvestedCount harvested + $revertedCount reverted = $($tpnRegistrationsToSave.Count) total" -Verbose
}
$newJson = @{
Registrations = $newRegistrations
Registrations = $registrationsToSave
'$schema' = "https://json.schemastore.org/component-detection-manifest.json"
} | ConvertTo-Json -depth 99
@@ -353,6 +438,149 @@ if ($Fix -and $registrationChanged) {
Set-GWVariable -Name CGMANIFEST_PATH -Value $cgManifestPath
}
# If -ForceHarvestedOnly was used, write the TPN manifest with filtered registrations
if ($Fix -and $ForceHarvestedOnly -and $tpnRegistrationsToSave.Count -gt 0) {
$tpnManifestDir = Join-Path -Path $PSScriptRoot -ChildPath "cgmanifest\tpn"
New-Item -ItemType Directory -Path $tpnManifestDir -Force | Out-Null
$tpnManifestPath = Join-Path -Path $tpnManifestDir -ChildPath "cgmanifest.json"
$tpnManifest = @{
Registrations = @($tpnRegistrationsToSave)
'$schema' = "https://json.schemastore.org/component-detection-manifest.json"
}
$tpnJson = $tpnManifest | ConvertTo-Json -depth 99
$tpnJson | Set-Content $tpnManifestPath -Encoding utf8NoBOM
Write-Verbose "TPN manifest created/updated with $($tpnRegistrationsToSave.Count) registrations (filtered for harvested packages)" -Verbose
}
# Skip legacy TPN update when -ForceHarvestedOnly already produced a filtered manifest
if ($Fix -and $registrationChanged -and -not $ForceHarvestedOnly) {
# Import ClearlyDefined module to check harvest status
Write-Verbose "Checking harvest status for newly added packages..." -Verbose
Import-Module -Name "$PSScriptRoot/clearlyDefined/src/ClearlyDefined" -Force
# Get harvest data for all registrations
$fullCgList = $newRegistrations |
ForEach-Object {
[PSCustomObject]@{
type = $_.Component.Type
Name = $_.Component.Nuget.Name
PackageVersion = $_.Component.Nuget.Version
}
}
$fullList = $fullCgList | Get-ClearlyDefinedData
$needHarvest = $fullList | Where-Object { !$_.harvested }
if ($needHarvest.Count -gt 0) {
Write-Verbose "Found $($needHarvest.Count) packages that need harvesting. Starting harvest..." -Verbose
$needHarvest | Select-Object -ExpandProperty coordinates | ConvertFrom-ClearlyDefinedCoordinates | Start-ClearlyDefinedHarvest
} else {
Write-Verbose "All packages are already harvested." -Verbose
}
# After manifest update and harvest, update TPN manifest with individual package status
Write-Verbose "Updating TPN manifest with individual package harvest status..." -Verbose
$tpnManifestDir = Join-Path -Path $PSScriptRoot -ChildPath "cgmanifest\tpn"
$tpnManifestPath = Join-Path -Path $tpnManifestDir -ChildPath "cgmanifest.json"
# Load current TPN manifest to get previous versions
$currentTpnManifest = @()
if (Test-Path $tpnManifestPath) {
$currentTpnJson = Get-Content $tpnManifestPath | ConvertFrom-Json -AsHashtable
$currentTpnManifest = $currentTpnJson.Registrations
}
# Build a lookup table of old versions
$oldVersions = @{}
foreach ($registration in $currentTpnManifest) {
$name = $registration.Component.Nuget.Name
if (!$oldVersions.ContainsKey($name)) {
$oldVersions[$name] = $registration
}
}
# Note: Do not recheck harvest status here. Harvesting is an async process that takes a significant amount of time.
# Use the harvest data from the initial check. Newly triggered harvests will be captured
# on the next run of this script after harvesting completes.
$finalHarvestData = $fullList
# Update packages individually based on harvest status
$tpnRegistrations = @()
$harvestedCount = 0
$restoredCount = 0
foreach ($item in $finalHarvestData) {
$matchingNewRegistration = $newRegistrations | Where-Object {
$_.Component.Nuget.Name -eq $item.Name -and
$_.Component.Nuget.Version -eq $item.PackageVersion
}
if ($matchingNewRegistration) {
if ($item.harvested) {
# Use new harvested version
$tpnRegistrations += $matchingNewRegistration
$harvestedCount++
} else {
# Package not harvested - find the last harvested version from ClearlyDefined API
Write-Verbose "Finding last harvested version for $($item.Name)..." -Verbose
$lastHarvestedVersion = $null
try {
# Search through all versions of this package to find the last harvested one
# Create a list of versions we know about from all runtimes
$packageVersionsToCheck = $newRegistrations | Where-Object {
$_.Component.Nuget.Name -eq $item.Name
} | ForEach-Object { $_.Component.Nuget.Version } | Sort-Object -Unique -Descending
foreach ($versionToCheck in $packageVersionsToCheck) {
$versionCheckList = [PSCustomObject]@{
type = "nuget"
Name = $item.Name
PackageVersion = $versionToCheck
}
$versionStatus = $versionCheckList | Get-ClearlyDefinedData
if ($versionStatus -and $versionStatus.harvested) {
$lastHarvestedVersion = $versionToCheck
break # Found the most recent harvested version
}
}
} catch {
Write-Verbose "Error checking harvested versions for $($item.Name): $_" -Verbose
}
# Use last harvested version if found, otherwise use old version as fallback
if ($lastHarvestedVersion) {
$revertedReg = New-NugetComponent -Name $item.Name -Version $lastHarvestedVersion -DevelopmentDependency:$matchingNewRegistration.DevelopmentDependency
$tpnRegistrations += $revertedReg
$restoredCount++
Write-Verbose "Reverted $($item.Name) from v$($item.PackageVersion) to last harvested v$lastHarvestedVersion" -Verbose
} elseif ($oldVersions.ContainsKey($item.Name)) {
$tpnRegistrations += $oldVersions[$item.Name]
$restoredCount++
Write-Verbose "Reverted $($item.Name) to previous version in TPN (no harvested version found)" -Verbose
} else {
Write-Warning "$($item.Name) v$($item.PackageVersion) not harvested and no harvested version found. Excluding from TPN manifest."
}
}
}
}
# Save updated TPN manifest
if ($tpnRegistrations.Count -gt 0) {
$tpnManifest = @{
Registrations = @($tpnRegistrations)
'$schema' = "https://json.schemastore.org/component-detection-manifest.json"
}
$tpnJson = $tpnManifest | ConvertTo-Json -depth 99
$tpnJson | Set-Content $tpnManifestPath -Encoding utf8NoBOM
Write-Verbose "TPN manifest updated: $harvestedCount new harvested + $restoredCount reverted to last harvested versions" -Verbose
}
}
if (!$Fix -and $registrationChanged) {
$temp = Get-GWTempPath
+1 -1
View File
@@ -4870,7 +4870,7 @@ function New-GlobalToolNupkgSource
}
# Set VSTS environment variable for CGManifest file path.
$globalToolCGManifestPFilePath = Join-Path -Path "$env:REPOROOT" -ChildPath "tools\cgmanifest.json"
$globalToolCGManifestPFilePath = Join-Path -Path "$env:REPOROOT" -ChildPath "tools/cgmanifest/main/cgmanifest.json"
$globalToolCGManifestFilePath = Resolve-Path -Path $globalToolCGManifestPFilePath -ErrorAction SilentlyContinue
if (($null -eq $globalToolCGManifestFilePath) -or (! (Test-Path -Path $globalToolCGManifestFilePath)))
{