From 57f4ccd1fcbf15a2c143007102fb81a79decf3ea Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 20 Oct 2021 13:11:13 -0700 Subject: [PATCH] add GH workflow to keep the cgmanifest up to date. --- .github/workflows/update-cgmanifest.yml | 52 ++++++ tools/findMissingNotices.ps1 | 214 +++++++++++++++++++----- 2 files changed, 222 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/update-cgmanifest.yml diff --git a/.github/workflows/update-cgmanifest.yml b/.github/workflows/update-cgmanifest.yml new file mode 100644 index 0000000000..14572aff2d --- /dev/null +++ b/.github/workflows/update-cgmanifest.yml @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update cgmanifest +on: + workflow_dispatch: + schedule: + # At 13:00 UTC every day. + - cron: '0 13 * * *' + +defaults: + run: + shell: pwsh + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + +jobs: + update-cgmanifest: + name: Update cgmanifest + timeout-minutes: 15 + runs-on: windows-latest + if: github.repository == 'PowerShell/PowerShell' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Execute script to update cgmanifest + run: | + Install-Module -Name dotnet.project.assets + - name: Execute script to update cgmanifest + run: | + ./tools/findMissingNotices.ps1 + - name: Microsoft Teams Notifier + uses: skitionek/notify-microsoft-teams@master + if: failure() + with: + webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }} + overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + id: cpr + if: env.CREATE_PR == 'true' + with: + commit-message: "Update the cgmanifest with missing or updated components" + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}" + reviewers: travisez13 + base: master + draft: false + branch: update-cgmanifest diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 7e802e4238..9d42f27093 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -1,9 +1,59 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# This script is used to completely rebuild the # Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13 -import-module dotnet.project.assets +Import-Module dotnet.project.assets + +$existingRegistrationTable = @{} +$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$existingRegistrationsJson.Registrations | ForEach-Object { + $registration = [Registration]$_ + $existingRegistrationTable.Add($registration.Component.Name(), $registration) +} + +# this function wraps native command Execution +# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ +function script:Start-NativeExecution { + param( + [scriptblock]$sb, + [switch]$IgnoreExitcode, + [switch]$VerboseOutputOnError + ) + $backupEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + if ($VerboseOutputOnError.IsPresent) { + $output = & $sb 2>&1 + } else { + & $sb + } + + # note, if $sb doesn't have a native invocation, $LASTEXITCODE will + # point to the obsolete value + if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { + if ($VerboseOutputOnError.IsPresent -and $output) { + $output | Out-String | Write-Verbose -Verbose + } + + # Get caller location for easier debugging + $caller = Get-PSCallStack -ErrorAction SilentlyContinue + if ($caller) { + $callerLocationParts = $caller[1].Location -split ":\s*line\s*" + $callerFile = $callerLocationParts[0] + $callerLine = $callerLocationParts[1] + + $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" + throw $errorMessage + } + throw "Execution of {$sb} failed with exit code $LASTEXITCODE" + } + } finally { + $ErrorActionPreference = $backupEAP + } +} + Class Registration { [Component]$Component [bool]$DevelopmentDependency @@ -79,58 +129,134 @@ function New-NugetComponent { return $registration } -$existingRegistrationTable = @{} -$newRegistrations = @() -$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable -$existingRegistrationsJson.Registrations | ForEach-Object { - $registration = [Registration]$_ - $existingRegistrationTable.Add($registration.Component.Name(), $registration) - $newRegistrations += $registration +$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' +if (!$IsWindows) { + $winDesktopSdk = 'Microsoft.NET.Sdk' + Write-Warning "Always using $winDesktopSdk since this is not windows!!!" } -Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive -Push-Location $PSScriptRoot\..\src\powershell-win-core -$null = dotnet restore -$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core -$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object { - $_.Type -eq 'package' -and - $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and - $_.Name -notlike 'StyleCop*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' -} | select-object -ExpandProperty name -Pop-Location -Get-PSDrive -Name pwsh-win-core | Remove-PSDrive +Function Get-CGRegistrations { + param( + [Parameter(Mandatory)] + [ValidateSet( + "alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86", + "modules")] + [string]$Runtime, -$updateRegistrations = @() -$targets | ForEach-Object { - $target = $_ - $parts = ($target -split '\|') - $name = $parts[0] - $targetVersion = $parts[1] - $pattern = [regex]::Escape($name) + " " - $tpnMatch = select-string -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern - if (!$tpnMatch) { - if ($existingRegistrationTable.ContainsKey($name)) { - $registrationVersion = $existingRegistrationTable.$name.Component.Version() - if ($registrationVersion -ne $targetVersion) { + [Parameter(Mandatory)] + [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable + ) + + $newRegistrations = $Registrations + + $dotnetTargetName = 'net6.0' + $dotnetTargetNameWin7 = 'net6.0-windows7.0' + $unixProjectName = 'powershell-unix' + $windowsProjectName = 'powershell-win-core' + $actualRuntime = $Runtime + + + + switch -regex ($Runtime) { + "alpine-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "linux-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "osx-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "win7-.*" { + $sdkToUse = $winDesktopSdk + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "win-.*" { + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "modules" { + $folder = "modules" + $actualRuntime = 'linux-x64' + $target = "$dotnetTargetName|$actualRuntime" + } + Default { + throw "Invalid runtime name: $Runtime" + } + } + Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + Push-Location $PSScriptRoot\..\src\$folder + try { + script:Start-NativeExecution -VerboseOutputOnError -sb { + dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse" + } + $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder + try { + $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { + $_.Type -eq 'package' -and + $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and + $_.Name -notlike 'StyleCop*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' + } | select-object -ExpandProperty name + } catch { + Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose + throw + } + } finally { + Pop-Location + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + } + + $targets | ForEach-Object { + $target = $_ + $parts = ($target -split '\|') + $name = $parts[0] + $targetVersion = $parts[1] + $pattern = [regex]::Escape($name) + " " + $tpnMatch = select-string -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern + + # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR + # the exisitng CG contains the registration, because if the existing CG contains the registration, + # that might be the only reason it is in the TPN. + if (!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) { + if (!$RegistrationTable.ContainsKey($target)) { $registration = New-NugetComponent -Name $name -Version $targetVersion - $updateRegistrations += $registration - } else { - Write-Verbose "$target already registered: $registrationVersion" -Verbose + $RegistrationTable.Add($target, $registration) } - } else { - $registration = New-NugetComponent -Name $name -Version $targetVersion - $newRegistrations += $registration } } } -if ($updateRegistrations.count -gt 0) { - #TODO delete existing and add new registration - throw "updating registrations is not implemented" +[System.Collections.Generic.Dictionary[string, Registration]]$registrations = @{} +$lastCount = 0 +foreach ($runtime in @("win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86")) { + Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations + $count = $registrations.Count + $newCount = $count - $lastCount + $lastCount = $count + Write-Verbose "$newCount new registrations, $count total..." -Verbose } -$newCount = $newRegistrations.count - $existingRegistrationTable.count +$newRegistrations = @() +foreach ($target in ($registrations.Keys | Sort-Object)) { + $newRegistrations += $registrations[$target] +} + +$count = $newRegistrations.Count @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json -Write-Verbose "$newCount registrations added" -Verbose +Write-Verbose "$count registrations created!" -Verbose