Strip authorization header on redirects with web cmdlets (#3885)

Invoke-WebRequest and Invoke-RestMethod cmdlets will now strip authorization header on redirect unless the new parameter `-PreserveAuthorizationOnRedirect` is specified.

The FullCLR implementation uses WebRequest to perform the request which silently strips the Authorization header when a redirect occurs.

The CoreCLR implementation uses HttpClient to perform the request which does not strip the authorization header. The change explicitly handles the initial redirect, removes the authorization header and submits the request to location in the response.

Fixes #2227

src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs

@@ -24,7 +24,7 @@ namespace Microsoft.PowerShell.Commands
 {
     /// <summary>
     /// Exception class for webcmdlets to enable returning HTTP error response
-    /// </summary>    
+    /// </summary>
     public sealed class HttpResponseException : HttpRequestException
     {
         /// <summary>
@@ -48,6 +48,22 @@ namespace Microsoft.PowerShell.Commands
     /// </summary>
     public abstract partial class WebRequestPSCmdlet : PSCmdlet
     {
+
+        /// <summary>
+        /// gets or sets the PreserveAuthorizationOnRedirect property
+        /// </summary>
+        /// <remarks>
+        /// This property overrides compatibility with web requests on Windows.
+        /// On FullCLR (WebRequest), authorization headers are stripped during redirect.
+        /// CoreCLR (HTTPClient) does not have this behavior so web requests that work on
+        /// PowerShell/FullCLR can fail with PowerShell/CoreCLR.  To provide compatibility,
+        /// we'll detect requests with an Authorization header and automatically strip
+        /// the header when the first redirect occurs. This switch turns off this logic for 
+        /// edge cases where the authorization header needs to be preserved across redirects.
+        /// </remarks>
+        [Parameter]
+        public virtual SwitchParameter PreserveAuthorizationOnRedirect { get; set; }
+
         #region Abstract Methods
 
         /// <summary>
@@ -111,7 +127,9 @@ namespace Microsoft.PowerShell.Commands
 
         #region Virtual Methods
 
-        internal virtual HttpClient GetHttpClient()
+        // NOTE: Only pass true for handleRedirect if the original request has an authorization header
+        // and PreserveAuthorizationOnRedirect is NOT set.
+        internal virtual HttpClient GetHttpClient(bool handleRedirect)
         {
             // By default the HttpClientHandler will automatically decompress GZip and Deflate content
             HttpClientHandler handler = new HttpClientHandler();
@@ -150,7 +168,12 @@ namespace Microsoft.PowerShell.Commands
                 handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
             }
 
-            if (WebSession.MaximumRedirection > -1)
+            // This indicates GetResponse will handle redirects.
+            if (handleRedirect)
+            {
+                handler.AllowAutoRedirect = false;
+            }
+            else if (WebSession.MaximumRedirection > -1)
             {
                 if (WebSession.MaximumRedirection == 0)
                 {
@@ -178,7 +201,7 @@ namespace Microsoft.PowerShell.Commands
             return httpClient;
         }
 
-        internal virtual HttpRequestMessage GetRequest(Uri uri)
+        internal virtual HttpRequestMessage GetRequest(Uri uri, bool stripAuthorization)
         {
             Uri requestUri = PrepareUri(uri);
             HttpMethod httpMethod = null;
@@ -217,6 +240,13 @@ namespace Microsoft.PowerShell.Commands
                     }
                     else
                     {
+                        if (stripAuthorization 
+                            && 
+                            String.Equals(entry.Key, HttpKnownHeaderNames.Authorization.ToString(), StringComparison.OrdinalIgnoreCase)
+                        )
+                        {
+                            continue;
+                        }
                         request.Headers.Add(entry.Key, entry.Value);
                     }
                 }
@@ -367,13 +397,77 @@ namespace Microsoft.PowerShell.Commands
             }
         }
 
-        internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request)
+        // Returns true if the status code is one of the supported redirection codes.
+        static bool IsRedirectCode(HttpStatusCode code)
+        {
+            int intCode = (int) code;
+            return
+            (
+                (intCode >= 300 && intCode < 304)
+                ||
+                intCode == 307
+            );
+        }
+
+        // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection.
+        // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness.
+        static bool IsRedirectToGet(HttpStatusCode code)
+        {
+            return
+            (
+                code == HttpStatusCode.Found
+                ||
+                code == HttpStatusCode.Moved
+                ||
+                code == HttpStatusCode.Redirect
+                ||
+                code == HttpStatusCode.RedirectMethod
+                ||
+                code == HttpStatusCode.TemporaryRedirect
+                ||
+                code == HttpStatusCode.RedirectKeepVerb
+                ||
+                code == HttpStatusCode.SeeOther
+            );
+        }
+
+        internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool stripAuthorization)
         {
             if (client == null) { throw new ArgumentNullException("client"); }
             if (request == null) { throw new ArgumentNullException("request"); }
 
             _cancelToken = new CancellationTokenSource();
-            return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
+            HttpResponseMessage response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
+
+            if (stripAuthorization && IsRedirectCode(response.StatusCode))
+            {
+                _cancelToken.Cancel();
+                _cancelToken = null;
+
+                // if explicit count was provided, reduce it for this redirection.
+                if (WebSession.MaximumRedirection > 0)
+                {
+                    WebSession.MaximumRedirection--;
+                }
+                // For selected redirects that used POST, GET must be used with the
+                // redirected Location.
+                // Since GET is the default; POST only occurs when -Method POST is used.
+                if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode))
+                {
+                    // See https://msdn.microsoft.com/en-us/library/system.net.httpstatuscode(v=vs.110).aspx
+                    Method = WebRequestMethod.Get;
+                }
+
+                // recreate the HttpClient with redirection enabled since the first call suppressed redirection
+                using (client = GetHttpClient(false))
+                using (HttpRequestMessage redirectRequest = GetRequest(response.Headers.Location, stripAuthorization:true))
+                {
+                    FillRequestStream(redirectRequest);
+                    _cancelToken = new CancellationTokenSource();
+                    response = client.SendAsync(redirectRequest, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
+                }
+            }
+            return response;
         }
 
         internal virtual void UpdateSession(HttpResponseMessage response)
@@ -396,7 +490,17 @@ namespace Microsoft.PowerShell.Commands
                 ValidateParameters();
                 PrepareSession();
 
-                using (HttpClient client = GetHttpClient())
+                // if the request contains an authorization header and PreserveAuthorizationOnRedirect is not set,
+                // it needs to be stripped on the first redirect.
+                bool stripAuthorization = null != WebSession
+                                          &&
+                                          null != WebSession.Headers
+                                          &&
+                                          !PreserveAuthorizationOnRedirect.IsPresent
+                                          &&
+                                          WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization.ToString());
+
+                using (HttpClient client = GetHttpClient(stripAuthorization))
                 {
                     int followedRelLink = 0;
                     Uri uri = Uri;
@@ -410,7 +514,7 @@ namespace Microsoft.PowerShell.Commands
                             WriteVerbose(linkVerboseMsg);
                         }
 
-                        using (HttpRequestMessage request = GetRequest(uri))
+                        using (HttpRequestMessage request = GetRequest(uri, stripAuthorization:false))
                         {
                             FillRequestStream(request);
                             try
@@ -426,7 +530,7 @@ namespace Microsoft.PowerShell.Commands
                                     requestContentLength);
                                 WriteVerbose(reqVerboseMsg);
 
-                                HttpResponseMessage response = GetResponse(client, request);
+                                HttpResponseMessage response = GetResponse(client, request, stripAuthorization);
 
                                 string contentType = ContentHelper.GetContentType(response);
                                 string respVerboseMsg = string.Format(CultureInfo.CurrentCulture,

test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1

@@ -143,6 +143,70 @@ function GetTestData
     return $body
 }
 
+function ExecuteRedirectRequest
+{
+    param (
+        [Parameter(Mandatory)]
+        [string]
+        $uri,
+
+        [ValidateSet('Invoke-WebRequest', 'Invoke-RestMethod')]
+        [string] $Cmdlet = 'Invoke-WebRequest',
+
+        [ValidateSet('POST', 'GET')]
+        [string] $Method = 'GET',
+
+        [switch] $PreserveAuthorizationOnRedirect
+    )
+    $result = [PSObject]@{Output = $null; Error = $null; Content = $null}
+
+    try
+    {
+        $headers = @{"Authorization" = "test"}
+        if ($Cmdlet -eq 'Invoke-WebRequest')
+        {
+            $result.Output = Invoke-WebRequest -Uri $uri -TimeoutSec 5 -Headers $headers -PreserveAuthorizationOnRedirect:$PreserveAuthorizationOnRedirect.IsPresent -Method $Method
+            $result.Content = $result.Output.Content | ConvertFrom-Json
+        }
+        else 
+        {            
+            $result.Output = Invoke-RestMethod -Uri $uri -TimeoutSec 5 -Headers $headers -PreserveAuthorizationOnRedirect:$PreserveAuthorizationOnRedirect.IsPresent -Method $Method
+            # NOTE: $result.Output should already be a PSObject (Invoke-RestMethod converts the returned json automatically)
+            # so simply reference $result.Output
+            $result.Content = $result.Output
+        }       
+    }
+    catch
+    {
+        $result.Error = $_
+    }
+
+    return $result
+}
+
+<#
+    Defines the list of redirect codes to test as well as the
+    expected Method when the redirection is handled.
+    See https://msdn.microsoft.com/en-us/library/windows/apps/system.net.httpstatuscode(v=vs.105).aspx 
+    for additonal details.
+#>
+$redirectTests = @(
+    @{redirectType = 'MultipleChoices'; redirectedMethod='POST'}
+    @{redirectType = 'Ambiguous'; redirectedMethod='POST'} # Synonym for MultipleChoices
+
+    @{redirectType = 'Moved'; redirectedMethod='GET'}
+    @{redirectType = 'MovedPermanently'; redirectedMethod='GET'} # Synonym for Moved
+
+    @{redirectType = 'Found'; redirectedMethod='GET'}
+    @{redirectType = 'Redirect'; redirectedMethod='GET'} # Synonym for Found
+
+    @{redirectType = 'redirectMethod'; redirectedMethod='GET'}
+    @{redirectType = 'SeeOther'; redirectedMethod='GET'} # Synonym for RedirectMethod
+
+    @{redirectType = 'TemporaryRedirect'; redirectedMethod='GET'}
+    @{redirectType = 'RedirectKeepVerb'; redirectedMethod='GET'} # Synonym for TemporaryRedirect
+)
+
 Describe "Invoke-WebRequest tests" -Tags "Feature" {
 
     BeforeAll {
@@ -223,7 +287,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
         $jsonContent.headers.Host | Should Match "httpbin.org"
         $jsonContent.headers.'User-Agent' | Should Match "WindowsPowerShell"
     }
-
+    
     It "Validate Invoke-WebRequest error for -MaximumRedirection" {
 
         $command = "Invoke-WebRequest -Uri 'http://httpbin.org/redirect/3' -MaximumRedirection 2 -TimeoutSec 5"
@@ -231,7 +295,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
         $result = ExecuteWebCommand -command $command
         $result.Error.FullyQualifiedErrorId | Should Be "WebCmdletWebResponseException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand"
     }
-
+    
     It "Invoke-WebRequest supports request that returns page containing UTF-8 data." {
 
         $command = "Invoke-WebRequest -Uri http://httpbin.org/encoding/utf8 -TimeoutSec 5"
@@ -545,6 +609,59 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
         $result.Output.RelationLink["last"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=3&linknumber=3"
     }
 
+    #region Redirect tests
+
+    It "Validates Invoke-WebRequest with -PreserveAuthorizationOnRedirect preserves the authorization header on redirect" -TestCases $redirectTests {
+        param($redirectType, $redirectedMethod)
+
+        $response = ExecuteRedirectRequest -Uri "http://localhost:8080/PowerShell?test=redirect&type=$redirectType" -PreserveAuthorizationOnRedirect
+        
+        $response.Error | Should BeNullOrEmpty
+        # ensure Authorization header has been preserved.
+        $response.Content.Headers -contains "Authorization" | Should Be $true
+    }
+
+
+    It "Validates Invoke-WebRequest preserves the authorization header on multiple redirects." -TestCases $redirectTests {
+        param($redirectType)
+
+        $response = ExecuteRedirectRequest -Uri "http://localhost:8080/PowerShell?test=redirect&type=$redirectType&multiredirect=true" -PreserveAuthorizationOnRedirect
+
+        $response.Error | Should BeNullOrEmpty
+        # ensure Authorization header was stripped
+        $response.Content.Headers -contains "Authorization" | Should Be $true
+    }
+
+    It "Validates Invoke-WebRequest strips the authorization header on various redirects" -TestCases $redirectTests {
+        param($redirectType)
+
+        $response = ExecuteRedirectRequest -Uri "http://localhost:8080/PowerShell?test=redirect&type=$redirectType"
+
+        $response.Error | Should BeNullOrEmpty
+        # ensure user-agent is present (i.e., no false positives )
+        $response.Content.Headers -contains "User-Agent" | Should Be $true
+        # ensure Authorization header has been removed.
+        $response.Content.Headers -contains "Authorization" | Should Be $false
+    }
+
+    # NOTE: Only testing redirection of POST -> GET for unique underlying values of HttpStatusCode.
+    # Some names overlap in underlying value.
+    It "Validates Invoke-WebRequest strips the authorization header redirects and switches from POST to GET when it handles the redirect" -TestCases $redirectTests {
+        param($redirectType, $redirectedMethod)
+
+        $response = ExecuteRedirectRequest -Uri "http://localhost:8080/PowerShell?test=redirect&type=$redirectType" -Method 'POST'
+        
+        $response.Error | Should BeNullOrEmpty
+        # ensure user-agent is present (i.e., no false positives )
+        $response.Content.Headers -contains "User-Agent" | Should Be $true
+        # ensure Authorization header has been removed.
+        $response.Content.Headers -contains "Authorization" | Should Be $false
+        # ensure POST was changed to GET for selected redirections and remains as POST for others.
+        $response.Content.HttpMethod | Should Be $redirectedMethod
+    } 
+
+    #endregion Redirect tests       
+
     BeforeEach {
         if ($env:http_proxy) {
             $savedHttpProxy = $env:http_proxy
@@ -955,6 +1072,58 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" {
         $result.Output.output | Should BeExactly 1
     }
 
+    #region Redirect tests
+
+    It "Validates Invoke-RestMethod with -PreserveAuthorizationOnRedirect preserves the authorization header on redirect" -TestCases $redirectTests {
+        param($redirectType, $redirectedMethod)
+
+        $response = ExecuteRedirectRequest  -Cmdlet 'Invoke-RestMethod' -Uri "http://localhost:8081/PowerShell?test=redirect&type=$redirectType" -PreserveAuthorizationOnRedirect
+        
+        $response.Error | Should BeNullOrEmpty
+        # ensure Authorization header has been preserved.
+        $response.Content.Headers -contains "Authorization" | Should Be $true
+    }
+
+    It "Validates Invoke-RestMethod preserves the authorization header on multiple redirects." -TestCases $redirectTests {
+        param($redirectType)
+
+        $response = ExecuteRedirectRequest  -Cmdlet 'Invoke-RestMethod' -Uri "http://localhost:8081/PowerShell?test=redirect&type=$redirectType&multiredirect=true" -PreserveAuthorizationOnRedirect
+
+        $response.Error | Should BeNullOrEmpty
+        # ensure Authorization header was stripped
+        $response.Content.Headers -contains "Authorization" | Should Be $true
+    }
+
+    It "Validates Invoke-RestMethod strips the authorization header on various redirects" -TestCases $redirectTests {
+        param($redirectType)
+
+        $response = ExecuteRedirectRequest  -Cmdlet 'Invoke-RestMethod' -Uri "http://localhost:8081/PowerShell?test=redirect&type=$redirectType"
+
+        $response.Error | Should BeNullOrEmpty
+        # ensure user-agent is present (i.e., no false positives )
+        $response.Output.Headers -contains "User-Agent" | Should Be $true
+        # ensure Authorization header has been removed.
+        $response.Content.Headers -contains "Authorization" | Should Be $false
+    }
+
+    # NOTE: Only testing redirection of POST -> GET for unique underlying values of HttpStatusCode.
+    # Some names overlap in underlying value.
+    It "Validates Invoke-RestMethod strips the authorization header redirects and switches from POST to GET when it handles the redirect" -TestCases $redirectTests {
+        param($redirectType, $redirectedMethod)
+
+        $response = ExecuteRedirectRequest  -Cmdlet 'Invoke-RestMethod' -Uri "http://localhost:8081/PowerShell?test=redirect&type=$redirectType" -Method 'POST'
+        
+        $response.Error | Should BeNullOrEmpty
+        # ensure user-agent is present (i.e., no false positives )
+        $response.Content.Headers -contains "User-Agent" | Should Be $true
+        # ensure Authorization header has been removed.
+        $response.Content.Headers -contains "Authorization" | Should Be $false
+        # ensure POST was changed to GET for selected redirections and remains as POST for others.
+        $response.Content.HttpMethod | Should Be $redirectedMethod
+    } 
+
+    #endregion Redirect tests       
+
     BeforeEach {
         if ($env:http_proxy) {
             $savedHttpProxy = $env:http_proxy

test/tools/Modules/HttpListener/HttpListener.psm1

@@ -46,6 +46,15 @@ Function Start-HTTPListener {
         [scriptblock]$script = {
             param ($Port)
 
+            $script:supportedRedirects = @{
+                [System.Net.HttpStatusCode]::Ok = 1; # No redirect
+                [System.Net.HttpStatusCode]::Found = 1;
+                [System.Net.HttpStatusCode]::MultipleChoices = 1;
+                [System.Net.HttpStatusCode]::Moved = 1;
+                [System.Net.HttpStatusCode]::SeeOther = 1;
+                [System.Net.HttpStatusCode]::TemporaryRedirect = 1;
+            }
+
             # HttpListener.QueryString is not being populated, need to follow-up with CoreFx, workaround is to parse it ourselves
             Function ParseQueryString([string]$url)
             {
@@ -92,6 +101,11 @@ Function Start-HTTPListener {
                     $test = $queryItems["test"]
                     Write-Verbose "Testing: $test"
 
+                    foreach ($key in $request.Headers.Keys)
+                    {
+                        Write-Verbose -Message "Found Header: $key, $($request.Headers[$key])"
+                    }
+                    
                     # the status code to return for the response
                     $statusCode = [System.Net.HttpStatusCode]::OK
                     # this is the body of the response, return json/xml as appropriate
@@ -120,21 +134,75 @@ Function Start-HTTPListener {
                             $contentType = $queryItems["contenttype"]
                             $output = $queryItems["output"]
                         }
+                        
+                        <# 
+                            This test provides support for multiple redirection types as well as a custom
+                            multi-hop redirection to handle Authorization stripping logic.
+                            The following  redirection types are supported:
+                                 MultipleChoices (300), Moved (301), Found (302), SeeOther (303), TemporaryRedirect (307)
+
+                            The original URL should  indicate the type of redirection.
+                            For example: The following indicates that a 302 redirection (found) should be used.
+                             ?test=redirectex&type=Found
+
+                            WebRequest cmdlet tests also use a special option called multiredirect. This produces two redirects
+                            where the second 
+
+                            Example: test=redirectex&type=Moved&multiredirect=true
+
+                            See also https://msdn.microsoft.com/en-us/library/system.net.httpstatuscode(v=vs.110).aspx
+                        #>
                         "redirect"
                         {
-                            $redirect = $queryItems["redirect"]
-                            if ($redirect -eq $null)
+                            $redirectedUrl = [string]::Empty
+                            $redirectType = $queryItems["type"]
+                            $multiredirect = $queryItems["multiredirect"]
+
+                            if ($redirectType -eq $null)
                             {
-                                $statusCode = [System.Net.HttpStatusCode]::Found
-                                $redirectedUrl = "${Url}?test=redirect&redirect=false"
-                                Write-Verbose "$redirectedUrl"
-                                $outputHeader.Add("Location",$redirectedUrl)
-                                Write-Verbose "Redirecting to $($outputHeader.Location)"
+                                # End of redirection
+                                $redirectType = 'Ok'
                             }
-                            else
+
+                            [System.Net.HttpStatusCode] $type = [System.Net.HttpStatusCode]::Found
+                            [bool] $isValid = [System.Enum]::TryParse($redirectType, $true, [ref] $type)
+                            if ($isValid -eq $false -or $script:supportedRedirects.ContainsKey($type) -eq $false)
                             {
-                                $output = $request | ConvertTo-Json
+                                Write-Verbose -Message "Invalid request type: $type"
+                                $statusCode = [System.Net.HttpStatusCode]::BadRequest
+                                $output = "Invalid Redirect Type: $type"
                             }
+                            elseif ($type -eq [System.Net.HttpStatusCode]::Ok)
+                            {
+                                # no redirection
+                                Write-Verbose -Message "No redirection"
+                                $output = $request | ConvertTo-Json -Depth 6
+                            }
+                            elseif ($multiredirect -eq $null)
+                            {
+                                Write-Verbose -Message "Standard redirection"
+                                $redirectedUrl = "${Url}?test=redirect&type=Ok"
+                            }
+                            elseif ($multiredirect -eq $true)
+                            {
+                               Write-Verbose -Message "Redirect 1 of 2"
+                               $redirectedUrl = "${Url}?test=redirect&type=$type&multiredirect=false"
+                            }
+                            elseif ($multiredirect -eq $false)
+                            {                      
+                                Write-Verbose -Message "Redirect 2 of 2"
+                                $redirectedUrl = "${Url}?test=redirect&type=$type"
+                            }
+
+                            if ($isValid)
+                            {
+                                $statusCode = $type
+                                if (-not [string]::IsNullOrEmpty($redirectedUrl))
+                                {
+                                    $outputHeader.Add("Location",$redirectedUrl)
+                                    Write-Verbose -Message "Redirecting to $($outputHeader.Location)" 
+                                }
+                            }                               
                         }
                         "linkheader"
                         {
@@ -189,11 +257,19 @@ Function Start-HTTPListener {
                     }
 
                     $response = $context.Response
-                    if ($contentType -ne $null)
+
+                    if ($outputHeader.ContainsKey('Content-Type') -eq $false)
                     {
-                        Write-Verbose "Setting ContentType to $contentType"
+                        if ([string]::IsNullOrEmpty($contentType))
+                        {
+                            $contentType = 'application/json'
+                        }
+                        
+                        $outputHeader.Add('Content-Type', $contentType)
                         $response.ContentType = $contentType
+                        Write-Verbose -Message "Setting ContentType to $contentType"
                     }
+
                     if ($statusCode -ne $null)
                     {
                         $response.StatusCode = $statusCode
@@ -203,6 +279,7 @@ Function Start-HTTPListener {
                     {
                         $response.Headers.Add($header, $outputHeader[$header])
                     }
+                    
                     if ($output -ne $null)
                     {
                         $buffer = [System.Text.Encoding]::UTF8.GetBytes($output)