From e3cf257a7d92b8e8ea863d5aa7c4f58c90a94251 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 14:44:19 -0700 Subject: [PATCH 1/8] Use hash to check for curl Our shebang is Bash. --- tools/download.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/download.sh b/tools/download.sh index fb3e6831e6..5694e9acce 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -20,13 +20,19 @@ case "$OSTYPE" in # Install curl and wget to download package case "$ID" in centos*) - if [[ -z $(command -v curl) ]]; then + if ! hash curl 2>/dev/null; then echo "curl not found, installing..." sudo yum install -y curl fi + version=rpm ;; ubuntu) + if ! hash curl 2>/dev/null; then + echo "curl not found, installing..." + sudo apt-get install -y curl + fi + case "$VERSION_ID" in 14.04) version=ubuntu1.14.04.1_amd64.deb @@ -38,10 +44,6 @@ case "$OSTYPE" in echo "Ubuntu $VERSION_ID is not supported!" >&2 exit 2 esac - if [[ -z $(command -v curl) ]]; then - echo "curl not found, installing..." - sudo apt-get install -y curl - fi ;; *) echo "$NAME is not supported!" >&2 @@ -49,6 +51,7 @@ case "$OSTYPE" in esac ;; darwin*) + # We don't check for curl as macOS should have a system version version=pkg ;; *) From 25eab9a2e5f05ceef014309ed0bced5191e01742 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 14:47:07 -0700 Subject: [PATCH 2/8] Quote all $variables For paranoia. --- tools/download.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/download.sh b/tools/download.sh index 5694e9acce..b6a6b1e42c 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -60,15 +60,15 @@ case "$OSTYPE" in ;; esac -info=$(get_info $version) +info=$(get_info "$version") # Parses $info for asset ID and package name -read asset package <<< $(echo $info | sed 's/[,"]//g' | awk '{ print $2; print $4 }') +read asset package <<< $(echo "$info" | sed 's/[,"]//g' | awk '{ print $2; print $4 }') # Downloads asset to file -packageuri=$(curl -s -i -H 'Accept: application/octet-stream' https://api.github.com/repos/PowerShell/PowerShell/releases/assets/$asset | +packageuri=$(curl -s -i -H 'Accept: application/octet-stream' "https://api.github.com/repos/PowerShell/PowerShell/releases/assets/$asset" | grep location | sed 's/location: //g') -curl -C - -s -o $package ${packageuri%$'\r'} +curl -C - -s -o "$package" ${packageuri%$'\r'} # Installs PowerShell package case "$OSTYPE" in @@ -91,7 +91,7 @@ case "$OSTYPE" in ;; esac echo "Installing $libicupackage, libunwind8, and $package with sudo ..." - sudo apt-get install -y libunwind8 $icupackage + sudo apt-get install -y libunwind8 "$icupackage" sudo dpkg -i "./$package" ;; *) @@ -99,14 +99,14 @@ case "$OSTYPE" in ;; darwin*) echo "Installing $package with sudo ..." - sudo installer -pkg ./$package -target / + sudo installer -pkg "./$package" -target / ;; esac powershell -noprofile -c '"Congratulations! PowerShell is installed at $PSHOME"' success=$? -if [[ $success != 0 ]]; then - echo "ERROR! PowerShell didn't install. Check script output" >&2 - exit $success +if [[ "$success" != 0 ]]; then + echo "ERROR: PowerShell failed to install!" >&2 + exit "$success" fi From 6b582851d672b40146905741234baadf452aea4f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 14:47:31 -0700 Subject: [PATCH 3/8] Don't download package silently Otherwise users think it's slow or hung. --- tools/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/download.sh b/tools/download.sh index b6a6b1e42c..c3c2c96fb0 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -68,7 +68,7 @@ read asset package <<< $(echo "$info" | sed 's/[,"]//g' | awk '{ print $2; print # Downloads asset to file packageuri=$(curl -s -i -H 'Accept: application/octet-stream' "https://api.github.com/repos/PowerShell/PowerShell/releases/assets/$asset" | grep location | sed 's/location: //g') -curl -C - -s -o "$package" ${packageuri%$'\r'} +curl -C - -o "$package" ${packageuri%$'\r'} # Installs PowerShell package case "$OSTYPE" in From e337d93720d7e87f82402da4ae8ca746f4f4068e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 14:48:32 -0700 Subject: [PATCH 4/8] Install OpenSSL via Homebrew on macOS Since it is required. Note that we do *not* unsafely link the OpenSSL libraries into the system folders, as we instead patch .NET Core's libraries to find OpenSSL in the installed location. --- tools/download.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/download.sh b/tools/download.sh index c3c2c96fb0..4a3cb58f4d 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -98,6 +98,19 @@ case "$OSTYPE" in esac ;; darwin*) + if hash brew 2>/dev/null; then + if [[ ! -d $(brew --prefix openssl) ]]; then + echo "Installing OpenSSL with brew..." + if ! brew install openssl; then + echo "ERROR: OpenSSL failed to install! Crypto functions will not work..." >&2 + # Don't abort because it is not fatal + fi + fi + else + echo "ERROR: brew not found! OpenSSL may not be available..." >&2 + # Don't abort because it is not fatal + fi + echo "Installing $package with sudo ..." sudo installer -pkg "./$package" -target / ;; From fc2e4e0e239e00052d6bd324a98cb18d7a092892 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 14:50:36 -0700 Subject: [PATCH 5/8] Revert "Temporarily disable OS X on Travis CI" This reverts commit bbf64d17ad8e0ad7dd7266408cd783d43ff37b07. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1670c08890..5635e6fd4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,10 @@ git: os: - linux + - osx sudo: required dist: trusty +osx_image: xcode7.3 addons: artifacts: From 363657717caaac6066b39c25882a47600d3b0138 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 15:19:53 -0700 Subject: [PATCH 6/8] Handle package download failure --- tools/download.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/download.sh b/tools/download.sh index 4a3cb58f4d..90ba9b02d6 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -70,6 +70,11 @@ packageuri=$(curl -s -i -H 'Accept: application/octet-stream' "https://api.githu grep location | sed 's/location: //g') curl -C - -o "$package" ${packageuri%$'\r'} +if [[ ! -r "$package" ]]; then + echo "ERROR: $package failed to download! Aborting..." >&2 + exit 1 +fi + # Installs PowerShell package case "$OSTYPE" in linux*) From 27530c403c2a1e1242e4dc59fca177e6abed9447 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 16:32:06 -0700 Subject: [PATCH 7/8] Use direct download URLs instead of API GitHub's API is throttled to 60 requests per hour per IP address when for non-authenticated calls, which was causing severe CI flakiness. While this adds another set of URLs to update for each release, the alternative was adding an OAuth token and maintaining its ownership. Moreover, this code is simpler than the previous API parsing. --- .travis.yml | 4 +++- tools/download.sh | 25 ++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5635e6fd4f..d0ffc8f17c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,9 @@ addons: paths: $(ls powershell*{deb,pkg} | tr "\n" ":") install: - - (cd tools && ./download.sh) + - pushd tools + - ./download.sh + - popd - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then rvm use 2.2.1; fi # Default 2.0.0 Ruby is buggy script: ./tools/travis.sh diff --git a/tools/download.sh b/tools/download.sh index 90ba9b02d6..ec1733f66a 100755 --- a/tools/download.sh +++ b/tools/download.sh @@ -7,10 +7,9 @@ trap ' kill -s INT "$$" ' INT -# Retrieves asset ID and package name of asset ending in argument -# $info looks like: "id": 1698239, "name": "powershell_0.4.0-1_amd64.deb", -get_info() { - curl -s https://api.github.com/repos/PowerShell/PowerShell/releases/latest | grep -B 1 "name.*$1" +get_url() { + release=v6.0.0-alpha.9 + echo "https://github.com/PowerShell/PowerShell/releases/download/$release/$1" } # Get OS specific asset ID and package name @@ -25,7 +24,7 @@ case "$OSTYPE" in sudo yum install -y curl fi - version=rpm + package=powershell-6.0.0_alpha.9-1.el7.centos.x86_64.rpm ;; ubuntu) if ! hash curl 2>/dev/null; then @@ -35,10 +34,10 @@ case "$OSTYPE" in case "$VERSION_ID" in 14.04) - version=ubuntu1.14.04.1_amd64.deb + package=powershell_6.0.0-alpha.9-1ubuntu1.14.04.1_amd64.deb ;; 16.04) - version=ubuntu1.16.04.1_amd64.deb + package=powershell_6.0.0-alpha.9-1ubuntu1.16.04.1_amd64.deb ;; *) echo "Ubuntu $VERSION_ID is not supported!" >&2 @@ -52,7 +51,7 @@ case "$OSTYPE" in ;; darwin*) # We don't check for curl as macOS should have a system version - version=pkg + package=powershell-6.0.0-alpha.9.pkg ;; *) echo "$OSTYPE is not supported!" >&2 @@ -60,15 +59,7 @@ case "$OSTYPE" in ;; esac -info=$(get_info "$version") - -# Parses $info for asset ID and package name -read asset package <<< $(echo "$info" | sed 's/[,"]//g' | awk '{ print $2; print $4 }') - -# Downloads asset to file -packageuri=$(curl -s -i -H 'Accept: application/octet-stream' "https://api.github.com/repos/PowerShell/PowerShell/releases/assets/$asset" | - grep location | sed 's/location: //g') -curl -C - -o "$package" ${packageuri%$'\r'} +curl -L -o "$package" $(get_url "$package") if [[ ! -r "$package" ]]; then echo "ERROR: $package failed to download! Aborting..." >&2 From 7c10dac20cb9e2d1d3d1a5aeab663d2b4e213552 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 2 Sep 2016 17:16:35 -0700 Subject: [PATCH 8/8] Update macOS install docs for OpenSSL --- docs/building/macos.md | 18 ++++---------- docs/installation/linux.md | 50 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/docs/building/macos.md b/docs/building/macos.md index 984c88481b..b5783deb9e 100644 --- a/docs/building/macos.md +++ b/docs/building/macos.md @@ -20,9 +20,8 @@ and use`Start-PSBootstrap` to install the dependencies. The `Start-PSBootstrap` function does the following: - Uses `brew` to install CMake, OpenSSL, and GNU WGet -- Links OpenSSL - Uninstalls any prior versions of .NET CLI -- Downloads and installs the latest .NET CLI 1.0.0-preview2 SDK to `~/.dotnet` +- Downloads and installs the latest .NET CLI 1.0.0-preview3 SDK to `~/.dotnet` If you want to use `dotnet` outside of `Start-PSBuild`, add `~/.dotnet` to your `PATH` environment variable. @@ -40,7 +39,7 @@ We cannot do this for you in the build module due to #[847][]. [847]: https://github.com/PowerShell/PowerShell/issues/847 error: dotnet restore -------------------------- +--------------------- If you run `dotnet restore` and get error like @@ -55,18 +54,11 @@ error: The type initializer for 'Crypto' threw an exception. error: The type initializer for 'CryptoInitializer' threw an exception. error: Unable to load DLL 'System.Security.Cryptography.Native': The specified module could not be found. error: (Exception from HRESULT: 0x8007007E) - ``` -Try the following - -* Make sure you have latest openssl and re-link it - -``` -brew update -brew install openssl -brew link --force openssl -``` +These means you did not use our `Start-PSBootstrap` function to setup your environment, +which handles patching .NET Core's bad cryptography libraries. +Please see our [macOS installation instructions](../installation/linux.md#openssl) for explanation. Build using our module ====================== diff --git a/docs/installation/linux.md b/docs/installation/linux.md index e2b5f3c3b8..636e4f4c88 100644 --- a/docs/installation/linux.md +++ b/docs/installation/linux.md @@ -72,9 +72,11 @@ sudo yum install https://github.com/PowerShell/PowerShell/releases/download/v6.0 [CentOS 7]: https://www.centos.org/download/ macOS 10.11 -========== +=========== -Using macOS 10.11, download the PKG package `powershell-6.0.0-alpha.9.pkg` from the [releases][] page onto the macOS machine. +Using macOS 10.11, download the PKG package +`powershell-6.0.0-alpha.9.pkg` +from the [releases][] page onto the macOS machine. Either double-click the file and follow the prompts, or install it from the terminal: @@ -83,6 +85,50 @@ or install it from the terminal: sudo installer -pkg powershell-6.0.0-alpha.9.pkg -target / ``` +OpenSSL +------- + +Also install [Homebrew's OpenSSL][openssl]: + +``` +brew install openssl +``` + +[Homebrew][brew] is the missing package manager for macOS. +If the `brew` command was not found, +you need to install Homebrew following [their instructions][brew]. + +.NET Core requires Homebrew's OpenSSL because the "OpenSSL" system libraries on macOS are not OpenSSL, +as Apple deprecated OpenSSL in favor of their own libraries. +This requirement is not a hard requirement for all of PowerShell; +however, most networking functions (such as `Invoke-WebRequest`) +do require OpenSSL to work properly. + +**Please ignore** .NET Core's installation instructions to manually link the OpenSSL libraries. +This is **not** required for PowerShell as we patch .NET Core's cryptography libraries to find Homebrew's OpenSSL in its installed location. +Again, **do not** run `brew link --force` nor `ln -s` for OpenSSL, regardless of other instructions. + +Homebrew previously allowed OpenSSL libraries to be linked to the system library location; +however, this created major security holes and is [no longer allowed][homebrew-patch]. +Because .NET Core's 1.0.0 release libraries still look in the prior system location for OpenSSL, +they will fail to work unless the libraries are manually placed there (security risk), +or their libraries are patched (which we do). +To patch .NET Core's cryptography libraries, we use `install_name_tool`: + +``` +find ~/.nuget -name System.Security.Cryptography.Native.dylib | xargs sudo install_name_tool -add_rpath /usr/local/opt/openssl/lib +``` + +This updates .NET Core's library to look in Homebrew's OpenSSL installation location instead of the system library location. +The PowerShell macOS package come with the necessary libraries patched, +and the build script patches the libraries on-the-fly when building from source. +You *can* run this command manually if you're having trouble with .NET Core's cryptography libraries. + + +[openssl]: https://github.com/Homebrew/homebrew-core/blob/master/Formula/openssl.rb +[brew]: http://brew.sh/ +[homebrew-patch]: https://github.com/Homebrew/brew/pull/597 + Paths =====