[feature] Make UTF-8 Default Encoding for application/json (#6109)

When a charset is not supplied for a JSON response, the default encoding should be UTF-8 per RFC 8259. This commit changes the default charset to UTF-8 for JSON responses when a charset is not defined.

src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs

@@ -165,6 +165,12 @@ namespace Microsoft.PowerShell.Commands
                 Encoding encoding = null;
                 // fill the Content buffer
                 string characterSet = WebResponseHelper.GetCharacterSet(BaseResponse);
+
+                if (String.IsNullOrEmpty(characterSet) && ContentHelper.IsJson(contentType))
+                {
+                    characterSet = Encoding.UTF8.HeaderName;
+                }
+
                 this.Content = StreamHelper.DecodeStream(RawContentStream, characterSet, out encoding);
                 this.Encoding = encoding;
             }

src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs

@@ -392,6 +392,11 @@ namespace Microsoft.PowerShell.Commands
                             StreamHelper.TryGetEncoding(charSet, out encoding);
                         }
 
+                        if (string.IsNullOrEmpty(charSet) && returnType == RestReturnType.Json)
+                        {
+                            encoding = Encoding.UTF8;
+                        }
+
                         object obj = null;
                         Exception ex = null;
 

test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1

@@ -992,6 +992,22 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
             $response.Output.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
             $response.Output | Should BeOfType 'Microsoft.PowerShell.Commands.BasicHtmlWebResponseObject'
         }
+
+        It "Verifies Invoke-WebRequest defaults to UTF8 on application/json when no charset is present" {
+            # when contenttype is set, WebListener suppresses charset unless it is included in the query
+            $query = @{
+                contenttype = 'application/json'
+                body        = '{"Test": "Test"}'
+            }
+            $uri = Get-WebListenerUrl -Test 'Response' -Query $query
+            $expectedEncoding = [System.Text.Encoding]::UTF8
+            $response = ExecuteWebRequest -Uri $uri -UseBasicParsing
+
+            $response.Error | Should BeNullOrEmpty
+            $response.Output.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
+            $response.Output | Should BeOfType 'Microsoft.PowerShell.Commands.BasicHtmlWebResponseObject'
+            $response.Output.Content | Should BeExactly $query.body
+        }
     }
 
     #endregion charset encoding tests
@@ -2302,6 +2318,21 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" {
             $response.Error | Should BeNullOrEmpty
             $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
         }
+
+        It "Verifies Invoke-RestMethod defaults to UTF8 on application/json when no charset is present" {
+            # when contenttype is set, WebListener suppresses charset unless it is included in the query
+            $query = @{
+                contenttype = 'application/json'
+                body        = '{"Test": "Test"}'
+            }
+            $uri = Get-WebListenerUrl -Test 'Response' -Query $query
+            $expectedEncoding = [System.Text.Encoding]::UTF8
+            $response = ExecuteRestMethod -Uri $uri -UseBasicParsing
+
+            $response.Error | Should BeNullOrEmpty
+            $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName
+            $response.output.Test | Should BeExactly 'Test'
+        }
     }
 
     #endregion charset encoding tests