diff --git a/.github/workflows/GHWorkflowHelper/GHWorkflowHelper.psm1 b/.github/workflows/GHWorkflowHelper/GHWorkflowHelper.psm1 new file mode 100644 index 0000000000..f0524ce6f2 --- /dev/null +++ b/.github/workflows/GHWorkflowHelper/GHWorkflowHelper.psm1 @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +function Set-GWVariable { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [Parameter(Mandatory = $true)] + [string]$Value + ) + + Write-Verbose "Setting CI variable $Name to $Value" -Verbose + + if ($env:GITHUB_ENV) { + "$Name=$Value" | Out-File $env:GITHUB_ENV -Append + } +} + +function Get-GWTempPath { + $temp = [System.IO.Path]::GetTempPath() + if ($env:RUNNER_TEMP) { + $temp = $env:RUNNER_TEMP + } + + Write-Verbose "Get CI Temp path: $temp" -Verbose + return $temp +} diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index a1c883c98f..bd1a352d0f 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -30,18 +30,16 @@ jobs: git fetch --prune --unshallow --tags - name: Execute Update .NET script run: | + Import-Module .github/workflows/GHWorkflowHelper $currentVersion = (Get-Content .\global.json | ConvertFrom-Json).sdk.version - Write-Verbose "OLD_VERSION=$currentVersion" -Verbose - "OLD_VERSION=$currentVersion" | Out-File $env:GITHUB_ENV -Append + Set-GWVariable -Name OLD_VERSION -Value $currentVersion ./tools/UpdateDotnetRuntime.ps1 -UpdateMSIPackaging -UseInternalFeed $newVersion = (Get-Content .\global.json | ConvertFrom-Json).sdk.version - Write-Verbose "NEW_VERSION=$newVersion" -Verbose - "NEW_VERSION=$newVersion" | Out-File $env:GITHUB_ENV -Append + Set-GWVariable -Name NEW_VERSION -Value $newVersion if ($currentVersion -ne $newVersion) { - Write-Verbose "CREATE_PR=true" -Verbose - "CREATE_PR=true" | Out-File $env:GITHUB_ENV -Append + Set-GWVariable -Name CREATE_PR -Value 'true' } - name: Microsoft Teams Notifier uses: skitionek/notify-microsoft-teams@master @@ -58,5 +56,26 @@ jobs: title: "Update .NET SDK version from `${{ env.OLD_VERSION }}` to `${{ env.NEW_VERSION }}`" base: master branch: dotnet_update - - + update-tpn: + name: Update Notices File + timeout-minutes: 15 + runs-on: windows-latest + if: github.repository == 'PowerShell/PowerShell' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Update Notices file + run: | + Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + id: cprtpn + with: + commit-message: "Update to the latest notice file" + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: "Update to the latest notice file" + reviewers: travisez13 + base: master + draft: false + branch: update-cgmanifest diff --git a/.github/workflows/update-cgmanifest.yml b/.github/workflows/update-cgmanifest.yml new file mode 100644 index 0000000000..13c3bada24 --- /dev/null +++ b/.github/workflows/update-cgmanifest.yml @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update cgmanifest +on: + workflow_dispatch: + pull_request: + branches: + - master + +defaults: + run: + shell: pwsh + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + CGMANIFEST_PATH: '' + NUGET_PACKAGES: ${{ github.workspace }}\.nuget\packages + +jobs: + update-cgmanifest: + name: Update cgmanifest + timeout-minutes: 15 + runs-on: windows-latest + if: github.repository == 'PowerShell/PowerShell' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Cache DotNet + uses: actions/cache@v2 + with: + path: | + ~\AppData\Local\Microsoft\dotnet + ${{ github.workspace }}\.nuget\packages + key: ${{ runner.os }}-${{ hashFiles('**\DotnetRuntimeMetadata.json') }}-${{ hashFiles('**\nuget.config') }} + - name: Sync tags + run: | + git fetch --prune --unshallow --tags + - name: Install Ships provider to deal with project.assets.json + run: | + Install-Module -Name dotnet.project.assets -force + - name: Bootstrap + run: | + Import-Module ./build.psm1 + Start-PSBootStrap + - name: Verify cgmanifest is up to date + run: | + Import-Module ./build.psm1 + Find-Dotnet + ./tools/findMissingNotices.ps1 + - name: Upload cgmanifest + uses: actions/upload-artifact@v2 + if: always() && env.CGMANIFEST_PATH != '' + with: + name: cgmanifest + path: ${{ env.CGMANIFEST_PATH }} diff --git a/build.psm1 b/build.psm1 index 62643a3e64..33a372d13d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -6,6 +6,8 @@ param( [parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false ) +. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1" + Set-StrictMode -Version 3.0 # On Unix paths is separated by colon @@ -2353,53 +2355,6 @@ function script:precheck([string]$command, [string]$missedMessage) { } } -# this function wraps native command Execution -# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ -function script:Start-NativeExecution -{ - param( - [scriptblock]$sb, - [switch]$IgnoreExitcode, - [switch]$VerboseOutputOnError - ) - $backupEAP = $ErrorActionPreference - $ErrorActionPreference = "Continue" - try { - if($VerboseOutputOnError.IsPresent) - { - $output = & $sb 2>&1 - } - else - { - & $sb - } - - # note, if $sb doesn't have a native invocation, $LASTEXITCODE will - # point to the obsolete value - if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { - if($VerboseOutputOnError.IsPresent -and $output) - { - $output | Out-String | Write-Verbose -Verbose - } - - # Get caller location for easier debugging - $caller = Get-PSCallStack -ErrorAction SilentlyContinue - if($caller) - { - $callerLocationParts = $caller[1].Location -split ":\s*line\s*" - $callerFile = $callerLocationParts[0] - $callerLine = $callerLocationParts[1] - - $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" - throw $errorMessage - } - throw "Execution of {$sb} failed with exit code $LASTEXITCODE" - } - } finally { - $ErrorActionPreference = $backupEAP - } -} - # Cleans the PowerShell repo - everything but the root folder function Clear-PSRepo { diff --git a/prquantifier.yaml b/prquantifier.yaml new file mode 100644 index 0000000000..240f660f88 --- /dev/null +++ b/prquantifier.yaml @@ -0,0 +1,11 @@ +# https://github.com/microsoft/PullRequestQuantifier/blob/main/docs/prquantifier-yaml.md +Excluded: +# defaults +- '*.csproj' +- prquantifier.yaml +- package-lock.json +- '*.md' +- '*.sln' +# autogenerated files +- cgmanifest.json +- assets/wix/files.wxs diff --git a/tools/buildCommon/startNativeExecution.ps1 b/tools/buildCommon/startNativeExecution.ps1 new file mode 100644 index 0000000000..ee7b00d04c --- /dev/null +++ b/tools/buildCommon/startNativeExecution.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# this function wraps native command Execution +# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ +function script:Start-NativeExecution { + param( + [Alias('sb')] + [Parameter(Mandatory=$true)] + [scriptblock]$ScriptBlock, + [switch]$IgnoreExitcode, + [switch]$VerboseOutputOnError + ) + + $backupEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + Write-Verbose "Executing: $ScriptBlock" + try { + if ($VerboseOutputOnError.IsPresent) { + $output = & $ScriptBlock 2>&1 + } else { + & $ScriptBlock + } + + # note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will + # point to the obsolete value + if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { + if ($VerboseOutputOnError.IsPresent -and $output) { + $output | Out-String | Write-Verbose -Verbose + } + + # Get caller location for easier debugging + $caller = Get-PSCallStack -ErrorAction SilentlyContinue + if ($caller) { + $callerLocationParts = $caller[1].Location -split ":\s*line\s*" + $callerFile = $callerLocationParts[0] + $callerLine = $callerLocationParts[1] + + $errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" + throw $errorMessage + } + throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE" + } + } finally { + $ErrorActionPreference = $backupEAP + } +} diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index d1cf198f8b..f66f88f129 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -1,9 +1,28 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# This script is used to completely rebuild the cgmanifgest.json file, +# which is used to generate the notice file. # Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13 -import-module dotnet.project.assets +param( + [switch] $Fix +) + +Import-Module dotnet.project.assets +Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force +. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" + +$existingRegistrationTable = @{} +$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$existingRegistrationsJson.Registrations | ForEach-Object { + $registration = [Registration]$_ + if ($registration.Component) { + $name = $registration.Component.Name() + $existingRegistrationTable.Add($name, $registration) + } +} + Class Registration { [Component]$Component [bool]$DevelopmentDependency @@ -56,10 +75,17 @@ Class Nuget { } } +$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' +if (!$IsWindows) { + $winDesktopSdk = 'Microsoft.NET.Sdk' + Write-Warning "Always using $winDesktopSdk since this is not windows!!!" +} + function New-NugetComponent { param( [string]$name, - [string]$version + [string]$version, + [switch]$DevelopmentDependency ) $nuget = [Nuget]@{ @@ -73,64 +99,188 @@ function New-NugetComponent { $registration = [Registration]@{ Component = $Component - DevelopmentDependency = $false + DevelopmentDependency = $DevelopmentDependency } return $registration } -$existingRegistrationTable = @{} -$newRegistrations = @() -$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable -$existingRegistrationsJson.Registrations | ForEach-Object { - $registration = [Registration]$_ - $existingRegistrationTable.Add($registration.Component.Name(), $registration) - $newRegistrations += $registration +$nugetPublicVersionCache = [System.Collections.Generic.Dictionary[string, string]]::new() +function Get-NuGetPublicVersion { + param( + [string]$Name, + [string]$Version + ) + + if($nugetPublicVersionCache.ContainsKey($Name)) { + return $nugetPublicVersionCache[$Name] + } + + try { + [System.Management.Automation.SemanticVersion]$desiredVersion = $Version + } catch { + [Version]$desiredVersion = $Version + } + + $publicVersion = $null + $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source nuget.org -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { + try { + $packageVersion = [System.Management.Automation.SemanticVersion]$_.Version + } catch { + # Fall back to using [version] if it is not a semantic version + $packageVersion = $_.Version + } + + $_ | Add-Member -Name SemVer -MemberType NoteProperty -Value $packageVersion -PassThru + } | Where-Object { $_.SemVer -le $desiredVersion } | Sort-Object -Property semver -Descending | Select-Object -First 1 -ExpandProperty Version + + if(!$publicVersion) { + Write-Warning "No public version found for $Name, using $Version" + $publicVersion = $Version + } + + if(!$nugetPublicVersionCache.ContainsKey($Name)) { + $nugetPublicVersionCache.Add($Name, $publicVersion) + } + + return $publicVersion } -Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive -Push-Location $PSScriptRoot\..\src\powershell-win-core -$null = dotnet restore -$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core -$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object { - $_.Type -eq 'package' -and - $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and - $_.Name -notlike 'StyleCop*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' -} | select-object -ExpandProperty name -Pop-Location -Get-PSDrive -Name pwsh-win-core | Remove-PSDrive +function Get-CGRegistrations { + param( + [Parameter(Mandatory)] + [ValidateSet( + "alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86", + "modules")] + [string]$Runtime, -$updateRegistrations = @() -$targets | ForEach-Object { - $target = $_ - $parts = ($target -split '\|') - $name = $parts[0] - $targetVersion = $parts[1] - $pattern = [regex]::Escape($name) + " " - $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern - if (!$tpnMatch) { - if ($existingRegistrationTable.ContainsKey($name)) { - $registrationVersion = $existingRegistrationTable.$name.Component.Version() - if ($registrationVersion -ne $targetVersion) { - $registration = New-NugetComponent -Name $name -Version $targetVersion - $updateRegistrations += $registration - } else { - Write-Verbose "$target already registered: $registrationVersion" -Verbose - } - } else { - $registration = New-NugetComponent -Name $name -Version $targetVersion - $newRegistrations += $registration + [Parameter(Mandatory)] + [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable + ) + + $registrationChanged = $false + + $dotnetTargetName = 'net6.0' + $dotnetTargetNameWin7 = 'net6.0-windows7.0' + $unixProjectName = 'powershell-unix' + $windowsProjectName = 'powershell-win-core' + $actualRuntime = $Runtime + + switch -regex ($Runtime) { + "alpine-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "linux-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "osx-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "win7-.*" { + $sdkToUse = $winDesktopSdk + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "win-.*" { + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "modules" { + $folder = "modules" + $actualRuntime = 'linux-x64' + $target = "$dotnetTargetName|$actualRuntime" + } + Default { + throw "Invalid runtime name: $Runtime" } } + + Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + Push-Location $PSScriptRoot\..\src\$folder + try { + Start-NativeExecution -VerboseOutputOnError -sb { + dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse" + } + $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder + try { + $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { $_.Type -eq 'package' } | select-object -ExpandProperty name + } catch { + Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose + throw + } + } finally { + Pop-Location + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + } + + $targets | ForEach-Object { + $target = $_ + $parts = ($target -split '\|') + $name = $parts[0] + $targetVersion = $parts[1] + $publicVersion = Get-NuGetPublicVersion -Name $name -Version $targetVersion + + # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR + # the exisitng CG contains the registration, because if the existing CG contains the registration, + # that might be the only reason it is in the TPN. + if (!$RegistrationTable.ContainsKey($target)) { + $DevelopmentDependency = $false + if (!$existingRegistrationTable.ContainsKey($name) -or $existingRegistrationTable.$name.Component.Version() -ne $publicVersion) { + $registrationChanged = $true + } + if ($existingRegistrationTable.ContainsKey($name) -and $existingRegistrationTable.$name.DevelopmentDependency) { + $DevelopmentDependency = $true + } + + $registration = New-NugetComponent -Name $name -Version $publicVersion -DevelopmentDependency:$DevelopmentDependency + $RegistrationTable.Add($target, $registration) + } + } + + return $registrationChanged } -if ($updateRegistrations.count -gt 0) { - #TODO delete existing and add new registration - throw "updating registrations is not implemented" +$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new() +$lastCount = 0 +$registrationChanged = $false +foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") { + $registrationChanged = (Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations) -or $registrationChanged + $count = $registrations.Count + $newCount = $count - $lastCount + $lastCount = $count + Write-Verbose "$newCount new registrations, $count total..." -Verbose } -$newCount = $newRegistrations.count - $existingRegistrationTable.count -@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json -Write-Verbose "$newCount registrations added" -Verbose +$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] } + +$count = $newRegistrations.Count +$newJson = @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 +if ($Fix -and $registrationChanged) { + $cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\cgmanifest.json).ProviderPath + $newJson | Set-Content $cgManifestPath + Set-GWVariable -Name CGMANIFEST_PATH -Value $cgManifestPath +} + +if (!$Fix -and $registrationChanged) { + $temp = Get-GWTempPath + + $tempJson = Join-Path -Path $temp -ChildPath "cgmanifest$((Get-Date).ToString('yyyMMddHHmm')).json" + $newJson | Set-Content $tempJson -Encoding utf8NoBOM + Set-GWVariable -Name CGMANIFEST_PATH -Value $tempJson + throw "cgmanifest is out of date. run ./tools/findMissingNotices.ps1 -Fix. Generated cgmanifest is here: $tempJson" +} + +Write-Verbose "$count registrations created!" -Verbose diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 70cf08fc79..2eab521a53 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -4211,7 +4211,7 @@ function Test-PackageManifest { } Process { - Write-Verbose "Processing $($man.files) files..." -verbose + Write-Verbose "Processing $($man.files.count) files..." -verbose $man.files | ForEach-Object { $filePath = Join-Path $PackagePath -childPath $_.fileName $checksumObj = $_.checksums | Where-Object {$_.algorithm -eq 'sha256'}