Update build script to always use 'dotnet publish' (#3589)

Also update 2 building docs

.spelling

@@ -167,6 +167,9 @@ SystemD
 andschwa's
  - docs/building/internals.md
 Catalog
+src
+powershell-unix
+MSBuild
  - docs/building/macos.md
 preview3
  - docs/community/governance.md

build.psm1

@@ -86,7 +86,6 @@ function Start-PSBuild {
         # to help avoid compilation error, because file are in use.
         [switch]$StopDevPowerShell,
 
-        [switch]$NoPath,
         [switch]$Restore,
         [string]$Output,
         [switch]$ResGen,
@@ -143,11 +142,6 @@ function Start-PSBuild {
         Stop-Process -Verbose
     }
 
-    if ($CrossGen -and !$Publish) {
-        # By specifying -CrossGen, we implicitly set -Publish to $true, if not already specified.
-        $Publish = $true
-    }
-
     if ($Clean) {
         log "Cleaning your working directory. You can also do it with 'git clean -fdX'"
         Push-Location $PSScriptRoot
@@ -217,7 +211,6 @@ function Start-PSBuild {
 
     # set output options
     $OptionsArguments = @{
-        Publish=$Publish
         CrossGen=$CrossGen
         Output=$Output
         FullCLR=$FullCLR
@@ -233,12 +226,7 @@ function Start-PSBuild {
     }
 
     # setup arguments
-    $Arguments = @()
-    if ($Publish -or $FullCLR) {
-        $Arguments += "publish"
-    } else {
-        $Arguments += "build"
-    }
+    $Arguments = @("publish")
     if ($Output) {
         $Arguments += "--output", $Output
     }
@@ -425,7 +413,7 @@ cmd.exe /C cd /d "$location" "&" "$($vcVarsPath)\vcvarsall.bat" "$NativeHostArch
 
     # add 'x' permission when building the standalone application
     # this is temporary workaround to a bug in dotnet.exe, tracking by dotnet/cli issue #6286
-    if ($Options.Configuration -eq "Linux" -and $Options.Publish) {
+    if ($Options.Configuration -eq "Linux") {
         chmod u+x $Options.Output
     }
 
@@ -506,8 +494,6 @@ function New-PSOptions {
                      "opensuse.42.1-x64")]
         [string]$Runtime,
 
-        [switch]$Publish,
-
         [switch]$CrossGen,
 
         [string]$Output,
@@ -599,32 +585,19 @@ function New-PSOptions {
 
     # Build the Output path
     if (!$Output) {
-        $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, $Runtime)
-
-        # Publish injects the publish directory
-        if ($Publish -or $FullCLR) {
-            $Output = [IO.Path]::Combine($Output, "publish")
-        }
-
-        $Output = [IO.Path]::Combine($Output, $Executable)
+        $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, $Runtime, "publish", $Executable)
     }
 
-    $RealFramework = $Framework
     if ($SMAOnly)
     {
         $Top = [IO.Path]::Combine($PSScriptRoot, "src", "System.Management.Automation")
-        if ($Framework -match 'netcoreapp')
-        {
-            $RealFramework = 'netstandard1.6'
-        }
     }
 
     return @{ Top = $Top;
               Configuration = $Configuration;
-              Framework = $RealFramework;
+              Framework = $Framework;
               Runtime = $Runtime;
               Output = $Output;
-              Publish = $Publish;
               CrossGen = $CrossGen }
 }
 

docs/building/internals.md

@@ -1,5 +1,4 @@
-Internals of build process
-==========================
+# Internals of build process
 
 The purpose of this document is to explain build process **internals** with subtle nuances.
 This document is not by any means complete.
@@ -8,8 +7,7 @@ The ultimate source of truth is the code in `.\build.psm1` that's getting execut
 This document assumes that you can successfully build PowerShell from sources for your platform.
 
 
-Top directory
--------------
+## Top directory
 
 We are calling `dotnet` tool build for `$Top` directory
 
@@ -19,22 +17,21 @@ We are calling `dotnet` tool build for `$Top` directory
 
 ### Dummy dependencies
 
-We use dummy dependencies between project.json files to leverage `dotnet` build functionality.
-For example, `src\powershell-win-core\project.json` has dependency on `Microsoft.PowerShell.PSReadLine`,
+We use dummy dependencies between projects to leverage `dotnet` build functionality.
+For example, `src\powershell-win-core\powershell-win-core.csproj` has dependency on `Microsoft.PowerShell.PSReadLine`,
 but in reality, there is no build dependency.
 
 Dummy dependencies allows us to build just `$Top` folder, instead of building several folders.
 
 ### Dummy dependencies rules
 
-* If assembly is part of FullCLR build,
-it should be listed as a dependency for FullCLR $Top folder (src\powershell-win-full)
+- If assembly is part of FullCLR build,
+  it should be listed as a dependency for FullCLR $Top folder (src\powershell-win-full)
 
-* If assembly is part of CoreCLR build,
-it should be listed as a dependency for $Top folder (src\powershell-unix or src\powershell-win-core)
+- If assembly is part of CoreCLR build,
+  it should be listed as a dependency for $Top folder (src\powershell-unix or src\powershell-win-core)
 
-Preliminary steps
------------------
+## Preliminary steps
 
 ### ResGen
 
@@ -45,18 +42,18 @@ it does *not* require PowerShell.
 The same command can be run manually:
 
 ```sh
-dotnet restore
 cd src/ResGen
+dotnet restore
 dotnet run
 ```
 
 Running the program does everything else:
 
-* for each project, given a `resources` folder
-  * creates a `gen` folder
-  * for each `*.resx` file
-    * fills in a strongly typed C# class
-    * writes it out to the corresponding `*.cs` file
+- for each project, given a `resources` folder
+  - creates a `gen` folder
+  - for each `*.resx` file
+    - fills in a strongly typed C# class
+    - writes it out to the corresponding `*.cs` file
 
 These files are *not* automatically updated on each build,
 as the project lacks the ability to detect changes.
@@ -76,20 +73,15 @@ Again, however, PowerShell is not required.
 The necessary steps can be run manually:
 
 ```sh
-dotnet restore
-cd src/TypeCatalogParser
-dotnet run
 cd ../TypeCatalogGen
+dotnet restore
 dotnet run ../Microsoft.PowerShell.CoreCLR.AssemblyLoadContext/CorePsTypeCatalog.cs powershell.inc
 ```
 
-The [`TypeCatalogParser`](../../src/TypeCatalogParser)
-parses the `src/Microsoft.PowerShell.SDK/project.lock.json` file
-(created by `dotnet restore`),
-which contains the necessary data to resolve the paths to the DLLs of each dependency of PowerShell.
-It produces a list of the location of all the DLLs that have types to be cataloged
-(the output file is `powershell.inc`).
-This list is taken as input to the [`TypeCatalogGen`](../../src/TypeCatalogGen) tool,
+The file `powershell.inc` is generated by running a custom MSBuild target,
+which can be found at [`build.sh`](../../build.sh#L15).
+`powershell.inc` contains the resolved paths to the DLLs of each dependency of PowerShell,
+and is taken as input to the [`TypeCatalogGen`](../../src/TypeCatalogGen) tool,
 which generates a source file `CorePsTypeCatalog.cs` for the `Microsoft.PowerShell.CoreCLR.AssemblyLoadContext` project.
 
 The error `The name 'InitializeTypeCatalog' does not exist in the current context`

docs/building/macos.md

@@ -1,16 +1,11 @@
-Build PowerShell on macOS
-========================
+# Build PowerShell on macOS
 
 This guide supplements the [Linux instructions](./linux.md), as
 building on macOS is almost identical.
 
-.NET Core (and by transitivity, us) only supports macOS 10.11, per
-CoreFX issue #[7731][].
+.NET Core 2.0 (and by transitivity, us) only supports macOS 10.12.
 
-[7731]: https://github.com/dotnet/corefx/issues/7731
-
-Environment
-===========
+## Environment
 
 You will want [Homebrew](http://brew.sh/), the missing package manager for macOS.
 Once installed, follow the same instructions to download and
@@ -21,53 +16,27 @@ The `Start-PSBootstrap` function does the following:
 
 - Uses `brew` to install CMake, OpenSSL, and GNU WGet
 - Uninstalls any prior versions of .NET CLI
-- Downloads and installs the latest .NET Core SDK 1.0.1 to `~/.dotnet`
+- Downloads and installs a preview version of .NET Core SDK 2.0 to `~/.dotnet`
 
 If you want to use `dotnet` outside of `Start-PSBuild`,
 add `~/.dotnet` to your `PATH` environment variable.
 
-error: Too many open files
---------------------------
+### error: Too many open files
 
 Due to a [bug][809] in NuGet, the `dotnet restore` command will fail without the limit increased.
 Run `ulimit -n 2048` to fix this in your session;
-add it your shell's profile to fix it permanently.
+add it to your shell's profile to fix it permanently.
 
 We cannot do this for you in the build module due to #[847][].
 
 [809]: https://github.com/dotnet/cli/issues/809
 [847]: https://github.com/PowerShell/PowerShell/issues/847
 
-error: dotnet restore
----------------------
-
-If you run `dotnet restore` and get error like
-
-```
-PS /Users/vors/dev/PowerShell> dotnet restore                                                                                         
-log  : Restoring packages for /Users/vors/dev/PowerShell/src/TypeCatalogGen/project.json...
-log  : Restoring packages for /Users/vors/dev/PowerShell/src/TypeCatalogParser/project.json...
-log  : Restoring packages for /Users/vors/dev/PowerShell/test/PSReadLine/project.json...
-log  : Restoring packages for /Users/vors/dev/PowerShell/test/csharp/project.json...
-error: Unable to load the service index for source http://www.myget.org/F/dotnet-core/api/v3/index.json.
-error:   The type initializer for 'Crypto' threw an exception.
-error:   The type initializer for 'CryptoInitializer' threw an exception.
-error:   Unable to load DLL 'System.Security.Cryptography.Native': The specified module could not be found.
-error:    (Exception from HRESULT: 0x8007007E)
-```
-
-These means you did not use our `Start-PSBootstrap` function to setup your environment,
-which handles patching .NET Core's bad cryptography libraries.
-Please see our [macOS installation instructions](../installation/linux.md#openssl) for explanation.
-
-Build using our module
-======================
+## Build using our module
 
 Instead of installing the Ubuntu package of PowerShell,
 download the `pkg` from our GitHub releases page using your browser, complete the wizard,
 start a `powershell` session, and use `Start-PSBuild` from the module.
 
-The output directory will be slightly different because your runtime identifier is different.
-PowerShell will be at `./src/powershell-unix/bin/Linux/netcoreapp1.1/osx.10.11-x64/powershell`,
-or `osx.10.10` depending on your operating system version.
+After building, PowerShell will be at `./src/powershell-unix/bin/Linux/netcoreapp2.0/osx.10.12-x64/publish/powershell`.
 Note that configuration is still `Linux` because it would be silly to make yet another separate configuration when it's used solely to work-around a CLI issue.

tools/appveyor.psm1

@@ -160,7 +160,7 @@ function Invoke-AppVeyorBuild
 
       if(Test-DailyBuild)
       {
-          Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore -Publish
+          Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore
       }
 
       ## Stop building 'FullCLR', but keep the parameters and related scripts for now.
@@ -287,7 +287,7 @@ function Invoke-AppVeyorTest
     #
     # CoreCLR
 
-    $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish -Configuration 'Release'))
+    $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Configuration 'Release'))
     Write-Host -Foreground Green 'Run CoreCLR tests'
     $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml"
     $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml"

tools/travis.ps1

@@ -97,7 +97,7 @@ $isFullBuild = $env:TRAVIS_EVENT_TYPE -eq 'cron' -or $env:TRAVIS_EVENT_TYPE -eq
 Write-Host -Foreground Green "Executing travis.ps1 `$isPR='$isPr' `$isFullBuild='$isFullBuild'"
 
 Start-PSBootstrap -Package:(-not $isPr)
-$output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish))
+$output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions))
 
 # CrossGen'ed assemblies cause a hang to happen intermittently when running powershell class
 # basic parsing tests in Linux/OSX. The hang seems to happen when generating dynamic assemblies.
@@ -111,7 +111,7 @@ $output = Split-Path -Parent (Get-PSOutput -Options (New-PSOptions -Publish))
 # without running those class parsing tests so as to avoid the hang.
 # NOTE: this change should be reverted once the 'CrossGen' issue is fixed by CoreCLR. The issue
 #       is tracked by https://github.com/dotnet/coreclr/issues/9745
-Start-PSBuild -CrossGen:$isFullBuild -Publish -PSModuleRestore
+Start-PSBuild -CrossGen:$isFullBuild -PSModuleRestore
 
 $pesterParam = @{ 'binDir' = $output }