From c6cd66bca8bc7e745c82afefd6ddd8493854aa8e Mon Sep 17 00:00:00 2001
From: Mark Kraus <github@markekraus.com>
Date: Wed, 21 Feb 2018 23:31:00 -0600
Subject: [PATCH] [feature] Make UTF-8 Default Encoding for application/json
 (#6109)

When a charset is not supplied for a JSON response, the default encoding should be UTF-8 per RFC 8259. This commit changes the default charset to UTF-8 for JSON responses when a charset is not defined.
---
 .../BasicHtmlWebResponseObject.Common.cs      |  6 ++++
 .../Common/InvokeRestMethodCommand.Common.cs  |  5 +++
 .../WebCmdlets.Tests.ps1                      | 31 +++++++++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs
index dfa6c16108..cc88ad4119 100644
--- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs
+++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs
@@ -165,6 +165,12 @@ namespace Microsoft.PowerShell.Commands
                 Encoding encoding = null;
                 // fill the Content buffer
                 string characterSet = WebResponseHelper.GetCharacterSet(BaseResponse);
+
+                if (String.IsNullOrEmpty(characterSet) && ContentHelper.IsJson(contentType))
+                {
+                    characterSet = Encoding.UTF8.HeaderName;
+                }
+
                 this.Content = StreamHelper.DecodeStream(RawContentStream, characterSet, out encoding);
                 this.Encoding = encoding;
             }
diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
index 0ca37557ea..69b251b627 100644
--- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
+++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
@@ -392,6 +392,11 @@ namespace Microsoft.PowerShell.Commands
                             StreamHelper.TryGetEncoding(charSet, out encoding);
                         }
 
+                        if (string.IsNullOrEmpty(charSet) && returnType == RestReturnType.Json)
+                        {
+                            encoding = Encoding.UTF8;
+                        }
+
                         object obj = null;
                         Exception ex = null;
 
diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
index 6bdde9c2a6..ccc52f142f 100644
--- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
+++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
@@ -992,6 +992,22 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
             $response.Output.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
             $response.Output | Should BeOfType 'Microsoft.PowerShell.Commands.BasicHtmlWebResponseObject'
         }
+
+        It "Verifies Invoke-WebRequest defaults to UTF8 on application/json when no charset is present" {
+            # when contenttype is set, WebListener suppresses charset unless it is included in the query
+            $query = @{
+                contenttype = 'application/json'
+                body        = '{"Test": "Test"}'
+            }
+            $uri = Get-WebListenerUrl -Test 'Response' -Query $query
+            $expectedEncoding = [System.Text.Encoding]::UTF8
+            $response = ExecuteWebRequest -Uri $uri -UseBasicParsing
+
+            $response.Error | Should BeNullOrEmpty
+            $response.Output.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
+            $response.Output | Should BeOfType 'Microsoft.PowerShell.Commands.BasicHtmlWebResponseObject'
+            $response.Output.Content | Should BeExactly $query.body
+        }
     }
 
     #endregion charset encoding tests
@@ -2302,6 +2318,21 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" {
             $response.Error | Should BeNullOrEmpty
             $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
         }
+
+        It "Verifies Invoke-RestMethod defaults to UTF8 on application/json when no charset is present" {
+            # when contenttype is set, WebListener suppresses charset unless it is included in the query
+            $query = @{
+                contenttype = 'application/json'
+                body        = '{"Test": "Test"}'
+            }
+            $uri = Get-WebListenerUrl -Test 'Response' -Query $query
+            $expectedEncoding = [System.Text.Encoding]::UTF8
+            $response = ExecuteRestMethod -Uri $uri -UseBasicParsing
+
+            $response.Error | Should BeNullOrEmpty
+            $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
+            $response.output.Test | Should BeExactly 'Test'
+        }
     }
 
     #endregion charset encoding tests