Merged PR 18402: Port CGmanifest fixes

This commit is contained in:
Travis Plunk
2021-12-08 20:35:02 +00:00
committed by Andrew Menagarishvili
parent 5d47dd4f50
commit ac30fcd388
8 changed files with 371 additions and 105 deletions
@@ -0,0 +1,27 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
function Set-GWVariable {
param(
[Parameter(Mandatory = $true)]
[string]$Name,
[Parameter(Mandatory = $true)]
[string]$Value
)
Write-Verbose "Setting CI variable $Name to $Value" -Verbose
if ($env:GITHUB_ENV) {
"$Name=$Value" | Out-File $env:GITHUB_ENV -Append
}
}
function Get-GWTempPath {
$temp = [System.IO.Path]::GetTempPath()
if ($env:RUNNER_TEMP) {
$temp = $env:RUNNER_TEMP
}
Write-Verbose "Get CI Temp path: $temp" -Verbose
return $temp
}
+27 -8
View File
@@ -30,18 +30,16 @@ jobs:
git fetch --prune --unshallow --tags
- name: Execute Update .NET script
run: |
Import-Module .github/workflows/GHWorkflowHelper
$currentVersion = (Get-Content .\global.json | ConvertFrom-Json).sdk.version
Write-Verbose "OLD_VERSION=$currentVersion" -Verbose
"OLD_VERSION=$currentVersion" | Out-File $env:GITHUB_ENV -Append
Set-GWVariable -Name OLD_VERSION -Value $currentVersion
./tools/UpdateDotnetRuntime.ps1 -UpdateMSIPackaging -UseInternalFeed
$newVersion = (Get-Content .\global.json | ConvertFrom-Json).sdk.version
Write-Verbose "NEW_VERSION=$newVersion" -Verbose
"NEW_VERSION=$newVersion" | Out-File $env:GITHUB_ENV -Append
Set-GWVariable -Name NEW_VERSION -Value $newVersion
if ($currentVersion -ne $newVersion) {
Write-Verbose "CREATE_PR=true" -Verbose
"CREATE_PR=true" | Out-File $env:GITHUB_ENV -Append
Set-GWVariable -Name CREATE_PR -Value 'true'
}
- name: Microsoft Teams Notifier
uses: skitionek/notify-microsoft-teams@master
@@ -58,5 +56,26 @@ jobs:
title: "Update .NET SDK version from `${{ env.OLD_VERSION }}` to `${{ env.NEW_VERSION }}`"
base: master
branch: dotnet_update
update-tpn:
name: Update Notices File
timeout-minutes: 15
runs-on: windows-latest
if: github.repository == 'PowerShell/PowerShell'
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Update Notices file
run: |
Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
id: cprtpn
with:
commit-message: "Update to the latest notice file"
committer: GitHub <noreply@github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
title: "Update to the latest notice file"
reviewers: travisez13
base: master
draft: false
branch: update-cgmanifest
+57
View File
@@ -0,0 +1,57 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
name: Update cgmanifest
on:
workflow_dispatch:
pull_request:
branches:
- master
defaults:
run:
shell: pwsh
env:
DOTNET_CLI_TELEMETRY_OPTOUT: 1
POWERSHELL_TELEMETRY_OPTOUT: 1
CGMANIFEST_PATH: ''
NUGET_PACKAGES: ${{ github.workspace }}\.nuget\packages
jobs:
update-cgmanifest:
name: Update cgmanifest
timeout-minutes: 15
runs-on: windows-latest
if: github.repository == 'PowerShell/PowerShell'
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Cache DotNet
uses: actions/cache@v2
with:
path: |
~\AppData\Local\Microsoft\dotnet
${{ github.workspace }}\.nuget\packages
key: ${{ runner.os }}-${{ hashFiles('**\DotnetRuntimeMetadata.json') }}-${{ hashFiles('**\nuget.config') }}
- name: Sync tags
run: |
git fetch --prune --unshallow --tags
- name: Install Ships provider to deal with project.assets.json
run: |
Install-Module -Name dotnet.project.assets -force
- name: Bootstrap
run: |
Import-Module ./build.psm1
Start-PSBootStrap
- name: Verify cgmanifest is up to date
run: |
Import-Module ./build.psm1
Find-Dotnet
./tools/findMissingNotices.ps1
- name: Upload cgmanifest
uses: actions/upload-artifact@v2
if: always() && env.CGMANIFEST_PATH != ''
with:
name: cgmanifest
path: ${{ env.CGMANIFEST_PATH }}
+2 -47
View File
@@ -6,6 +6,8 @@ param(
[parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false
)
. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1"
Set-StrictMode -Version 3.0
# On Unix paths is separated by colon
@@ -2353,53 +2355,6 @@ function script:precheck([string]$command, [string]$missedMessage) {
}
}
# this function wraps native command Execution
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
function script:Start-NativeExecution
{
param(
[scriptblock]$sb,
[switch]$IgnoreExitcode,
[switch]$VerboseOutputOnError
)
$backupEAP = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
if($VerboseOutputOnError.IsPresent)
{
$output = & $sb 2>&1
}
else
{
& $sb
}
# note, if $sb doesn't have a native invocation, $LASTEXITCODE will
# point to the obsolete value
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
if($VerboseOutputOnError.IsPresent -and $output)
{
$output | Out-String | Write-Verbose -Verbose
}
# Get caller location for easier debugging
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
if($caller)
{
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
$callerFile = $callerLocationParts[0]
$callerLine = $callerLocationParts[1]
$errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
throw $errorMessage
}
throw "Execution of {$sb} failed with exit code $LASTEXITCODE"
}
} finally {
$ErrorActionPreference = $backupEAP
}
}
# Cleans the PowerShell repo - everything but the root folder
function Clear-PSRepo
{
+11
View File
@@ -0,0 +1,11 @@
# https://github.com/microsoft/PullRequestQuantifier/blob/main/docs/prquantifier-yaml.md
Excluded:
# defaults
- '*.csproj'
- prquantifier.yaml
- package-lock.json
- '*.md'
- '*.sln'
# autogenerated files
- cgmanifest.json
- assets/wix/files.wxs
@@ -0,0 +1,47 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# this function wraps native command Execution
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
function script:Start-NativeExecution {
param(
[Alias('sb')]
[Parameter(Mandatory=$true)]
[scriptblock]$ScriptBlock,
[switch]$IgnoreExitcode,
[switch]$VerboseOutputOnError
)
$backupEAP = $ErrorActionPreference
$ErrorActionPreference = "Continue"
Write-Verbose "Executing: $ScriptBlock"
try {
if ($VerboseOutputOnError.IsPresent) {
$output = & $ScriptBlock 2>&1
} else {
& $ScriptBlock
}
# note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will
# point to the obsolete value
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
if ($VerboseOutputOnError.IsPresent -and $output) {
$output | Out-String | Write-Verbose -Verbose
}
# Get caller location for easier debugging
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
if ($caller) {
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
$callerFile = $callerLocationParts[0]
$callerLine = $callerLocationParts[1]
$errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
throw $errorMessage
}
throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE"
}
} finally {
$ErrorActionPreference = $backupEAP
}
}
+199 -49
View File
@@ -1,9 +1,28 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# This script is used to completely rebuild the cgmanifgest.json file,
# which is used to generate the notice file.
# Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13
import-module dotnet.project.assets
param(
[switch] $Fix
)
Import-Module dotnet.project.assets
Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force
. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1"
$existingRegistrationTable = @{}
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
$existingRegistrationsJson.Registrations | ForEach-Object {
$registration = [Registration]$_
if ($registration.Component) {
$name = $registration.Component.Name()
$existingRegistrationTable.Add($name, $registration)
}
}
Class Registration {
[Component]$Component
[bool]$DevelopmentDependency
@@ -56,10 +75,17 @@ Class Nuget {
}
}
$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop'
if (!$IsWindows) {
$winDesktopSdk = 'Microsoft.NET.Sdk'
Write-Warning "Always using $winDesktopSdk since this is not windows!!!"
}
function New-NugetComponent {
param(
[string]$name,
[string]$version
[string]$version,
[switch]$DevelopmentDependency
)
$nuget = [Nuget]@{
@@ -73,64 +99,188 @@ function New-NugetComponent {
$registration = [Registration]@{
Component = $Component
DevelopmentDependency = $false
DevelopmentDependency = $DevelopmentDependency
}
return $registration
}
$existingRegistrationTable = @{}
$newRegistrations = @()
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
$existingRegistrationsJson.Registrations | ForEach-Object {
$registration = [Registration]$_
$existingRegistrationTable.Add($registration.Component.Name(), $registration)
$newRegistrations += $registration
$nugetPublicVersionCache = [System.Collections.Generic.Dictionary[string, string]]::new()
function Get-NuGetPublicVersion {
param(
[string]$Name,
[string]$Version
)
if($nugetPublicVersionCache.ContainsKey($Name)) {
return $nugetPublicVersionCache[$Name]
}
try {
[System.Management.Automation.SemanticVersion]$desiredVersion = $Version
} catch {
[Version]$desiredVersion = $Version
}
$publicVersion = $null
$publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source nuget.org -AllVersions -ErrorAction SilentlyContinue | ForEach-Object {
try {
$packageVersion = [System.Management.Automation.SemanticVersion]$_.Version
} catch {
# Fall back to using [version] if it is not a semantic version
$packageVersion = $_.Version
}
$_ | Add-Member -Name SemVer -MemberType NoteProperty -Value $packageVersion -PassThru
} | Where-Object { $_.SemVer -le $desiredVersion } | Sort-Object -Property semver -Descending | Select-Object -First 1 -ExpandProperty Version
if(!$publicVersion) {
Write-Warning "No public version found for $Name, using $Version"
$publicVersion = $Version
}
if(!$nugetPublicVersionCache.ContainsKey($Name)) {
$nugetPublicVersionCache.Add($Name, $publicVersion)
}
return $publicVersion
}
Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive
Push-Location $PSScriptRoot\..\src\powershell-win-core
$null = dotnet restore
$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core
$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object {
$_.Type -eq 'package' -and
$_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
$_.Name -notlike 'StyleCop*' -and
$_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
$_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
} | select-object -ExpandProperty name
Pop-Location
Get-PSDrive -Name pwsh-win-core | Remove-PSDrive
function Get-CGRegistrations {
param(
[Parameter(Mandatory)]
[ValidateSet(
"alpine-x64",
"linux-arm",
"linux-arm64",
"linux-x64",
"osx-arm64",
"osx-x64",
"win-arm",
"win-arm64",
"win7-x64",
"win7-x86",
"modules")]
[string]$Runtime,
$updateRegistrations = @()
$targets | ForEach-Object {
$target = $_
$parts = ($target -split '\|')
$name = $parts[0]
$targetVersion = $parts[1]
$pattern = [regex]::Escape($name) + " "
$tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
if (!$tpnMatch) {
if ($existingRegistrationTable.ContainsKey($name)) {
$registrationVersion = $existingRegistrationTable.$name.Component.Version()
if ($registrationVersion -ne $targetVersion) {
$registration = New-NugetComponent -Name $name -Version $targetVersion
$updateRegistrations += $registration
} else {
Write-Verbose "$target already registered: $registrationVersion" -Verbose
}
} else {
$registration = New-NugetComponent -Name $name -Version $targetVersion
$newRegistrations += $registration
[Parameter(Mandatory)]
[System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable
)
$registrationChanged = $false
$dotnetTargetName = 'net6.0'
$dotnetTargetNameWin7 = 'net6.0-windows7.0'
$unixProjectName = 'powershell-unix'
$windowsProjectName = 'powershell-win-core'
$actualRuntime = $Runtime
switch -regex ($Runtime) {
"alpine-.*" {
$folder = $unixProjectName
$target = "$dotnetTargetName|$Runtime"
}
"linux-.*" {
$folder = $unixProjectName
$target = "$dotnetTargetName|$Runtime"
}
"osx-.*" {
$folder = $unixProjectName
$target = "$dotnetTargetName|$Runtime"
}
"win7-.*" {
$sdkToUse = $winDesktopSdk
$folder = $windowsProjectName
$target = "$dotnetTargetNameWin7|$Runtime"
}
"win-.*" {
$folder = $windowsProjectName
$target = "$dotnetTargetNameWin7|$Runtime"
}
"modules" {
$folder = "modules"
$actualRuntime = 'linux-x64'
$target = "$dotnetTargetName|$actualRuntime"
}
Default {
throw "Invalid runtime name: $Runtime"
}
}
Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
Push-Location $PSScriptRoot\..\src\$folder
try {
Start-NativeExecution -VerboseOutputOnError -sb {
dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse"
}
$null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder
try {
$targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { $_.Type -eq 'package' } | select-object -ExpandProperty name
} catch {
Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose
throw
}
} finally {
Pop-Location
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
}
$targets | ForEach-Object {
$target = $_
$parts = ($target -split '\|')
$name = $parts[0]
$targetVersion = $parts[1]
$publicVersion = Get-NuGetPublicVersion -Name $name -Version $targetVersion
# Add the registration to the cgmanifest if the TPN does not contain the name of the target OR
# the exisitng CG contains the registration, because if the existing CG contains the registration,
# that might be the only reason it is in the TPN.
if (!$RegistrationTable.ContainsKey($target)) {
$DevelopmentDependency = $false
if (!$existingRegistrationTable.ContainsKey($name) -or $existingRegistrationTable.$name.Component.Version() -ne $publicVersion) {
$registrationChanged = $true
}
if ($existingRegistrationTable.ContainsKey($name) -and $existingRegistrationTable.$name.DevelopmentDependency) {
$DevelopmentDependency = $true
}
$registration = New-NugetComponent -Name $name -Version $publicVersion -DevelopmentDependency:$DevelopmentDependency
$RegistrationTable.Add($target, $registration)
}
}
return $registrationChanged
}
if ($updateRegistrations.count -gt 0) {
#TODO delete existing and add new registration
throw "updating registrations is not implemented"
$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new()
$lastCount = 0
$registrationChanged = $false
foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") {
$registrationChanged = (Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations) -or $registrationChanged
$count = $registrations.Count
$newCount = $count - $lastCount
$lastCount = $count
Write-Verbose "$newCount new registrations, $count total..." -Verbose
}
$newCount = $newRegistrations.count - $existingRegistrationTable.count
@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json
Write-Verbose "$newCount registrations added" -Verbose
$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] }
$count = $newRegistrations.Count
$newJson = @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99
if ($Fix -and $registrationChanged) {
$cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\cgmanifest.json).ProviderPath
$newJson | Set-Content $cgManifestPath
Set-GWVariable -Name CGMANIFEST_PATH -Value $cgManifestPath
}
if (!$Fix -and $registrationChanged) {
$temp = Get-GWTempPath
$tempJson = Join-Path -Path $temp -ChildPath "cgmanifest$((Get-Date).ToString('yyyMMddHHmm')).json"
$newJson | Set-Content $tempJson -Encoding utf8NoBOM
Set-GWVariable -Name CGMANIFEST_PATH -Value $tempJson
throw "cgmanifest is out of date. run ./tools/findMissingNotices.ps1 -Fix. Generated cgmanifest is here: $tempJson"
}
Write-Verbose "$count registrations created!" -Verbose
+1 -1
View File
@@ -4211,7 +4211,7 @@ function Test-PackageManifest {
}
Process {
Write-Verbose "Processing $($man.files) files..." -verbose
Write-Verbose "Processing $($man.files.count) files..." -verbose
$man.files | ForEach-Object {
$filePath = Join-Path $PackagePath -childPath $_.fileName
$checksumObj = $_.checksums | Where-Object {$_.algorithm -eq 'sha256'}