From 79a1f803092abf4cbd07718d774070c9aa7a81e4 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 18 Apr 2017 13:12:35 -0700 Subject: [PATCH] Update build script to always use 'dotnet publish' (#3589) Also update 2 building docs --- .spelling | 3 +++ build.psm1 | 35 ++++----------------------- docs/building/internals.md | 48 ++++++++++++++++---------------------- docs/building/macos.md | 47 +++++++------------------------------ tools/appveyor.psm1 | 4 ++-- tools/travis.ps1 | 4 ++-- 6 files changed, 39 insertions(+), 102 deletions(-) diff --git a/.spelling b/.spelling index 59384fb1e3..1644f77dd4 100644 --- a/.spelling +++ b/.spelling @@ -167,6 +167,9 @@ SystemD andschwa's - docs/building/internals.md Catalog +src +powershell-unix +MSBuild - docs/building/macos.md preview3 - docs/community/governance.md diff --git a/build.psm1 b/build.psm1 index 5108a3e874..45888639ff 100644 --- a/build.psm1 +++ b/build.psm1 @@ -86,7 +86,6 @@ function Start-PSBuild { # to help avoid compilation error, because file are in use. [switch]$StopDevPowerShell, - [switch]$NoPath, [switch]$Restore, [string]$Output, [switch]$ResGen, @@ -143,11 +142,6 @@ function Start-PSBuild { Stop-Process -Verbose } - if ($CrossGen -and !$Publish) { - # By specifying -CrossGen, we implicitly set -Publish to $true, if not already specified. - $Publish = $true - } - if ($Clean) { log "Cleaning your working directory. You can also do it with 'git clean -fdX'" Push-Location $PSScriptRoot @@ -217,7 +211,6 @@ function Start-PSBuild { # set output options $OptionsArguments = @{ - Publish=$Publish CrossGen=$CrossGen Output=$Output FullCLR=$FullCLR @@ -233,12 +226,7 @@ function Start-PSBuild { } # setup arguments - $Arguments = @() - if ($Publish -or $FullCLR) { - $Arguments += "publish" - } else { - $Arguments += "build" - } + $Arguments = @("publish") if ($Output) { $Arguments += "--output", $Output } @@ -425,7 +413,7 @@ cmd.exe /C cd /d "$location" "&" "$($vcVarsPath)\vcvarsall.bat" "$NativeHostArch # add 'x' permission when building the standalone application # this is temporary workaround to a bug in dotnet.exe, tracking by dotnet/cli issue #6286 - if ($Options.Configuration -eq "Linux" -and $Options.Publish) { + if ($Options.Configuration -eq "Linux") { chmod u+x $Options.Output } @@ -506,8 +494,6 @@ function New-PSOptions { "opensuse.42.1-x64")] [string]$Runtime, - [switch]$Publish, - [switch]$CrossGen, [string]$Output, @@ -599,32 +585,19 @@ function New-PSOptions { # Build the Output path if (!$Output) { - $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, $Runtime) - - # Publish injects the publish directory - if ($Publish -or $FullCLR) { - $Output = [IO.Path]::Combine($Output, "publish") - } - - $Output = [IO.Path]::Combine($Output, $Executable) + $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, $Runtime, "publish", $Executable) } - $RealFramework = $Framework if ($SMAOnly) { $Top = [IO.Path]::Combine($PSScriptRoot, "src", "System.Management.Automation") - if ($Framework -match 'netcoreapp') - { - $RealFramework = 'netstandard1.6' - } } return @{ Top = $Top; Configuration = $Configuration; - Framework = $RealFramework; + Framework = $Framework; Runtime = $Runtime; Output = $Output; - Publish = $Publish; CrossGen = $CrossGen } } diff --git a/docs/building/internals.md b/docs/building/internals.md index 53ecf9953e..c8c93076e3 100644 --- a/docs/building/internals.md +++ b/docs/building/internals.md @@ -1,5 +1,4 @@ -Internals of build process -========================== +# Internals of build process The purpose of this document is to explain build process **internals** with subtle nuances. This document is not by any means complete. @@ -8,8 +7,7 @@ The ultimate source of truth is the code in `.\build.psm1` that's getting execut This document assumes that you can successfully build PowerShell from sources for your platform. -Top directory -------------- +## Top directory We are calling `dotnet` tool build for `$Top` directory @@ -19,22 +17,21 @@ We are calling `dotnet` tool build for `$Top` directory ### Dummy dependencies -We use dummy dependencies between project.json files to leverage `dotnet` build functionality. -For example, `src\powershell-win-core\project.json` has dependency on `Microsoft.PowerShell.PSReadLine`, +We use dummy dependencies between projects to leverage `dotnet` build functionality. +For example, `src\powershell-win-core\powershell-win-core.csproj` has dependency on `Microsoft.PowerShell.PSReadLine`, but in reality, there is no build dependency. Dummy dependencies allows us to build just `$Top` folder, instead of building several folders. ### Dummy dependencies rules -* If assembly is part of FullCLR build, -it should be listed as a dependency for FullCLR $Top folder (src\powershell-win-full) +- If assembly is part of FullCLR build, + it should be listed as a dependency for FullCLR $Top folder (src\powershell-win-full) -* If assembly is part of CoreCLR build, -it should be listed as a dependency for $Top folder (src\powershell-unix or src\powershell-win-core) +- If assembly is part of CoreCLR build, + it should be listed as a dependency for $Top folder (src\powershell-unix or src\powershell-win-core) -Preliminary steps ------------------ +## Preliminary steps ### ResGen @@ -45,18 +42,18 @@ it does *not* require PowerShell. The same command can be run manually: ```sh -dotnet restore cd src/ResGen +dotnet restore dotnet run ``` Running the program does everything else: -* for each project, given a `resources` folder - * creates a `gen` folder - * for each `*.resx` file - * fills in a strongly typed C# class - * writes it out to the corresponding `*.cs` file +- for each project, given a `resources` folder + - creates a `gen` folder + - for each `*.resx` file + - fills in a strongly typed C# class + - writes it out to the corresponding `*.cs` file These files are *not* automatically updated on each build, as the project lacks the ability to detect changes. @@ -76,20 +73,15 @@ Again, however, PowerShell is not required. The necessary steps can be run manually: ```sh -dotnet restore -cd src/TypeCatalogParser -dotnet run cd ../TypeCatalogGen +dotnet restore dotnet run ../Microsoft.PowerShell.CoreCLR.AssemblyLoadContext/CorePsTypeCatalog.cs powershell.inc ``` -The [`TypeCatalogParser`](../../src/TypeCatalogParser) -parses the `src/Microsoft.PowerShell.SDK/project.lock.json` file -(created by `dotnet restore`), -which contains the necessary data to resolve the paths to the DLLs of each dependency of PowerShell. -It produces a list of the location of all the DLLs that have types to be cataloged -(the output file is `powershell.inc`). -This list is taken as input to the [`TypeCatalogGen`](../../src/TypeCatalogGen) tool, +The file `powershell.inc` is generated by running a custom MSBuild target, +which can be found at [`build.sh`](../../build.sh#L15). +`powershell.inc` contains the resolved paths to the DLLs of each dependency of PowerShell, +and is taken as input to the [`TypeCatalogGen`](../../src/TypeCatalogGen) tool, which generates a source file `CorePsTypeCatalog.cs` for the `Microsoft.PowerShell.CoreCLR.AssemblyLoadContext` project. The error `The name 'InitializeTypeCatalog' does not exist in the current context` diff --git a/docs/building/macos.md b/docs/building/macos.md index b5b53058e9..980c622548 100644 --- a/docs/building/macos.md +++ b/docs/building/macos.md @@ -1,16 +1,11 @@ -Build PowerShell on macOS -======================== +# Build PowerShell on macOS This guide supplements the [Linux instructions](./linux.md), as building on macOS is almost identical. -.NET Core (and by transitivity, us) only supports macOS 10.11, per -CoreFX issue #[7731][]. +.NET Core 2.0 (and by transitivity, us) only supports macOS 10.12. -[7731]: https://github.com/dotnet/corefx/issues/7731 - -Environment -=========== +## Environment You will want [Homebrew](http://brew.sh/), the missing package manager for macOS. Once installed, follow the same instructions to download and @@ -21,53 +16,27 @@ The `Start-PSBootstrap` function does the following: - Uses `brew` to install CMake, OpenSSL, and GNU WGet - Uninstalls any prior versions of .NET CLI -- Downloads and installs the latest .NET Core SDK 1.0.1 to `~/.dotnet` +- Downloads and installs a preview version of .NET Core SDK 2.0 to `~/.dotnet` If you want to use `dotnet` outside of `Start-PSBuild`, add `~/.dotnet` to your `PATH` environment variable. -error: Too many open files --------------------------- +### error: Too many open files Due to a [bug][809] in NuGet, the `dotnet restore` command will fail without the limit increased. Run `ulimit -n 2048` to fix this in your session; -add it your shell's profile to fix it permanently. +add it to your shell's profile to fix it permanently. We cannot do this for you in the build module due to #[847][]. [809]: https://github.com/dotnet/cli/issues/809 [847]: https://github.com/PowerShell/PowerShell/issues/847 -error: dotnet restore ---------------------- - -If you run `dotnet restore` and get error like - -``` -PS /Users/vors/dev/PowerShell> dotnet restore -log : Restoring packages for /Users/vors/dev/PowerShell/src/TypeCatalogGen/project.json... -log : Restoring packages for /Users/vors/dev/PowerShell/src/TypeCatalogParser/project.json... -log : Restoring packages for /Users/vors/dev/PowerShell/test/PSReadLine/project.json... -log : Restoring packages for /Users/vors/dev/PowerShell/test/csharp/project.json... -error: Unable to load the service index for source http://www.myget.org/F/dotnet-core/api/v3/index.json. -error: The type initializer for 'Crypto' threw an exception. -error: The type initializer for 'CryptoInitializer' threw an exception. -error: Unable to load DLL 'System.Security.Cryptography.Native': The specified module could not be found. -error: (Exception from HRESULT: 0x8007007E) -``` - -These means you did not use our `Start-PSBootstrap` function to setup your environment, -which handles patching .NET Core's bad cryptography libraries. -Please see our [macOS installation instructions](../installation/linux.md#openssl) for explanation. - -Build using our module -====================== +## Build using our module Instead of installing the Ubuntu package of PowerShell, download the `pkg` from our GitHub releases page using your browser, complete the wizard, start a `powershell` session, and use `Start-PSBuild` from the module. -The output directory will be slightly different because your runtime identifier is different. -PowerShell will be at `./src/powershell-unix/bin/Linux/netcoreapp1.1/osx.10.11-x64/powershell`, -or `osx.10.10` depending on your operating system version. +After building, PowerShell will be at `./src/powershell-unix/bin/Linux/netcoreapp2.0/osx.10.12-x64/publish/powershell`. Note that configuration is still `Linux` because it would be silly to make yet another separate configuration when it's used solely to work-around a CLI issue. diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 697da69ac1..09e98b86fb 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -160,7 +160,7 @@ function Invoke-AppVeyorBuild if(Test-DailyBuild) { - Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore -Publish + Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore } ## Stop building 'FullCLR', but keep the parameters and related scripts for now. @@ -287,7 +287,7 @@ function Invoke-AppVeyorTest # # CoreCLR - $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish -Configuration 'Release')) + $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Configuration 'Release')) Write-Host -Foreground Green 'Run CoreCLR tests' $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml" $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml" diff --git a/tools/travis.ps1 b/tools/travis.ps1 index 68716e560a..7bff805a03 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -97,7 +97,7 @@ $isFullBuild = $env:TRAVIS_EVENT_TYPE -eq 'cron' -or $env:TRAVIS_EVENT_TYPE -eq Write-Host -Foreground Green "Executing travis.ps1 `$isPR='$isPr' `$isFullBuild='$isFullBuild'" Start-PSBootstrap -Package:(-not $isPr) -$output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish)) +$output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions)) # CrossGen'ed assemblies cause a hang to happen intermittently when running powershell class # basic parsing tests in Linux/OSX. The hang seems to happen when generating dynamic assemblies. @@ -111,7 +111,7 @@ $output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish)) # without running those class parsing tests so as to avoid the hang. # NOTE: this change should be reverted once the 'CrossGen' issue is fixed by CoreCLR. The issue # is tracked by https://github.com/dotnet/coreclr/issues/9745 -Start-PSBuild -CrossGen:$isFullBuild -Publish -PSModuleRestore +Start-PSBuild -CrossGen:$isFullBuild -PSModuleRestore $pesterParam = @{ 'binDir' = $output }