From d20d53eaacafd4645ea668bd46e62093d3f3e503 Mon Sep 17 00:00:00 2001 From: Mark Kraus Date: Mon, 26 Mar 2018 12:23:24 -0700 Subject: [PATCH] Add -Resume Feature to Web Cmdlets (#6447) Fixes #5964 Adds -Resume switch to Invoke-WebRequest and Invoke-RestMethod -Resume requires -OutFile Enables the ability to resume downloading a partially or incompletely downloaded file. File Size is the only indicator of local and remote file parity. If the local file is smaller than the remote file and the remote endpoint supports resume, the local file will be appended with the remaining bytes. If the local file is larger than the remote file, the local file will be overwritten If the remote server does not support resume, the local file will be overwritten If the local file is the same size as the remote file, the remote endpoint will return a 416 status code. This response is special-cased as a success in this instance. The local file remains untouched and it is assumed the file was already successfully downloaded previously. If the local file does not exist it will be created and the entire remote file will be requested. Added tests for all code new code paths (I'm pretty sure anyway) Added /Resume Controller to WebListener Documented /Resume Controller Updated .spelling to reflect terms in WebListener docs Note: I had to change the way GetResponse() tracks the current URI as we now have 3 places where the call is taking place. I don't foresee this causing any regressions. This area needs some refactoring. especially if we want to implement a retry mechanism --- .spelling | 2 + .../Common/WebRequestPSCmdlet.Common.cs | 109 ++++++++- .../utility/WebCmdlet/StreamHelper.cs | 15 +- .../resources/WebCmdletStrings.resx | 8 +- .../WebCmdlets.Tests.ps1 | 218 ++++++++++++++++++ .../Modules/WebListener/WebListener.psm1 | 1 + test/tools/WebListener/Constants.cs | 5 - .../Controllers/ResponseController.cs | 2 +- .../Controllers/ResumeController.cs | 126 ++++++++++ test/tools/WebListener/README.md | 69 ++++++ test/tools/WebListener/Startup.cs | 4 + 11 files changed, 547 insertions(+), 12 deletions(-) create mode 100644 test/tools/WebListener/Controllers/ResumeController.cs diff --git a/.spelling b/.spelling index 77cc97a19f..64e31b6317 100644 --- a/.spelling +++ b/.spelling @@ -1104,6 +1104,8 @@ v6.0. #region test/tools/WebListener/README.md Overrides - test/tools/WebListener/README.md Auth +NoResume NTLM +NumberBytes ResponseHeaders #endregion diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index f5c3cd3d32..ba03ffe246 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -354,6 +354,12 @@ namespace Microsoft.PowerShell.Commands [Parameter] public virtual SwitchParameter PassThru { get; set; } + /// + /// Resumes downloading a partial or incomplete file. OutFile is required. + /// + [Parameter] + public virtual SwitchParameter Resume { get; set; } + #endregion #endregion Virtual Properties @@ -512,7 +518,15 @@ namespace Microsoft.PowerShell.Commands if (PassThru && (OutFile == null)) { ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing, - "WebCmdletOutFileMissingException"); + "WebCmdletOutFileMissingException", nameof(PassThru)); + ThrowTerminatingError(error); + } + + // Resume requires OutFile. + if (Resume.IsPresent && OutFile == null) + { + ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing, + "WebCmdletOutFileMissingException", nameof(Resume)); ThrowTerminatingError(error); } } @@ -637,6 +651,14 @@ namespace Microsoft.PowerShell.Commands get { return (!ShouldSaveToOutFile || PassThru); } } + /// + /// Determines whether writing to a file should Resume and append rather than overwrite. + /// + internal bool ShouldResume + { + get { return (Resume.IsPresent && _resumeSuccess); } + } + #endregion Helper Properties #region Helper Methods @@ -860,6 +882,16 @@ namespace Microsoft.PowerShell.Commands /// internal int _maximumFollowRelLink = Int32.MaxValue; + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + private HttpMethod GetHttpMethod(WebRequestMethod method) { switch (Method) @@ -1062,6 +1094,22 @@ namespace Microsoft.PowerShell.Commands } } + // If the file to resume downloading exists, create the Range request header using the file size. + // If not, create a Range to request the entire file. + if (Resume.IsPresent) + { + var fileInfo = new FileInfo(QualifiedOutFile); + if (fileInfo.Exists) + { + request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); + _resumeFileSize = fileInfo.Length; + } + else + { + request.Headers.Range = new RangeHeaderValue(0, null); + } + } + // Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent // Default behavior is continue to send body content anyway after a short period // Here it send the two part as a whole. @@ -1233,6 +1281,9 @@ namespace Microsoft.PowerShell.Commands if (client == null) { throw new ArgumentNullException("client"); } if (request == null) { throw new ArgumentNullException("request"); } + // Track the current URI being used by various requests and re-requests. + var currentUri = request.RequestUri; + _cancelToken = new CancellationTokenSource(); HttpResponseMessage response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); @@ -1256,14 +1307,51 @@ namespace Microsoft.PowerShell.Commands } // recreate the HttpClient with redirection enabled since the first call suppressed redirection + currentUri = new Uri(request.RequestUri, response.Headers.Location); using (client = GetHttpClient(false)) - using (HttpRequestMessage redirectRequest = GetRequest(new Uri(request.RequestUri, response.Headers.Location), stripAuthorization:true)) + using (HttpRequestMessage redirectRequest = GetRequest(currentUri, stripAuthorization:true)) { FillRequestStream(redirectRequest); _cancelToken = new CancellationTokenSource(); response = client.SendAsync(redirectRequest, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); } } + + // Request again without the Range header because the server indicated the range was not satisfiable. + // This happens when the local file is larger than the remote file. + // If the size of the remote file is the same as the local file, there is nothing to resume. + if (Resume.IsPresent && + response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && + (response.Content.Headers.ContentRange.HasLength && + response.Content.Headers.ContentRange.Length != _resumeFileSize)) + { + _cancelToken.Cancel(); + + WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); + + // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() + // are treated as a standard -OutFile request. This also disables appending local file. + Resume = new SwitchParameter(false); + + using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri, stripAuthorization:false)) + { + FillRequestStream(requestWithoutRange); + long requestContentLength = 0; + if (requestWithoutRange.Content != null) + requestContentLength = requestWithoutRange.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + requestWithoutRange.Method, + requestWithoutRange.RequestUri, + requestContentLength); + WriteVerbose(reqVerboseMsg); + + return GetResponse(client, requestWithoutRange, stripAuthorization); + } + } + + _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; return response; } @@ -1336,7 +1424,22 @@ namespace Microsoft.PowerShell.Commands contentType); WriteVerbose(respVerboseMsg); - if (!response.IsSuccessStatusCode) + bool _isSuccess = response.IsSuccessStatusCode; + + // Check if the Resume range was not satisfiable because the file already completed downloading. + // This happens when the local file is the same size as the remote file. + if (Resume.IsPresent && + response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && + response.Content.Headers.ContentRange.HasLength && + response.Content.Headers.ContentRange.Length == _resumeFileSize) + { + _isSuccess = true; + WriteVerbose(String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.OutFileWritingSkipped, OutFile)); + // Disable writing to the OutFile. + OutFile = null; + } + + if (!_isSuccess) { string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure, (int)response.StatusCode, response.ReasonPhrase); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index cd02e22533..5c60c4058a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -342,9 +342,20 @@ namespace Microsoft.PowerShell.Commands /// internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet) { - using (FileStream output = File.Create(filePath)) + // If the web cmdlet should resume, append the file instead of overwriting. + if(cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume) { - WriteToStream(stream, output, cmdlet); + using (FileStream output = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.Read)) + { + WriteToStream(stream, output, cmdlet); + } + } + else + { + using (FileStream output = File.Create(filePath)) + { + WriteToStream(stream, output, cmdlet); + } } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index 82958bbe76..f22655aaf5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -187,7 +187,10 @@ Path '{0}' is not a file system path. Please specify the path to a file in the file system. - The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the PassThru parameter, then retry. + The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the {0} parameter, then retry. + + + The file will not be re-downloaded because the remote file is the same size as the OutFile: {0} The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry. @@ -249,6 +252,9 @@ {0} {1} with {2}-byte payload + + The remote server indicated it could not resume downloading. The local file will be overwritten. + received {0}-byte response of content type {1} diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index 350cc1f07c..d60a2187f8 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1618,6 +1618,116 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { } } + Context "Invoke-WebRequest File Resume Feature" { + BeforeAll { + $outFile = Join-Path $TestDrive "resume.txt" + + # Download the entire file to reference in tests + $referenceFile = Join-Path $TestDrive "reference.txt" + $resumeUri = Get-WebListenerUrl -Test 'Resume' + Invoke-WebRequest -uri $resumeUri -OutFile $referenceFile -ErrorAction Stop + $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile + $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length + } + + AfterEach { + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + } + + It "Invoke-WebRequest -Resume requires -OutFile" { + { Invoke-WebRequest -Resume -Uri $resumeUri -ErrorAction Stop } | + Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand' + } + + It "Invoke-WebRequest -Resume Downloads the whole file when the file does not exist" { + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $response.StatusCode | Should -Be 206 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-WebRequest -Resume overwrites an existing file that is larger than the remote file" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'false' + $response.StatusCode | Should -Be 200 + $response.Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-WebRequest -Resume overwrites existing file when remote server does not support resume" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $response = Invoke-WebRequest -Uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $response.StatusCode | Should -Be 200 + $response.Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-WebRequest -Resume resumes downloading from bytes" -TestCases @( + @{bytes = 4} + @{bytes = 8} + @{bytes = 12} + @{bytes = 16} + ) { + param($bytes, $statuscode) + # Simulate partial download + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue "Bytes/$bytes" + $null = Invoke-WebRequest -uri $uri -OutFile $outFile + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes + + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $response.StatusCode | Should -Be 206 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-WebRequest -Resume assumes the file was successfully completed when the local and remote file are the same size." { + # Download the entire file + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $null = Invoke-WebRequest -uri $uri -OutFile $outFile + $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + # The web cmdlets special case 416 as a success code when the local file and remote file are the same size + $response.StatusCode | Should -Be 416 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" + } + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy @@ -2779,6 +2889,114 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { } } + Context "Invoke-RestMethod File Resume Feature" { + BeforeAll { + $outFile = Join-Path $TestDrive "resume.txt" + + # Download the entire file to reference in tests + $referenceFile = Join-Path $TestDrive "reference.txt" + $resumeUri = Get-WebListenerUrl -Test 'Resume' + Invoke-RestMethod -uri $resumeUri -OutFile $referenceFile -ErrorAction Stop + $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile + $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length + } + + AfterEach { + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + } + + It "Invoke-RestMethod -Resume requires -OutFile" { + { Invoke-RestMethod -Resume -Uri $resumeUri -ErrorAction Stop } | + Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeRestMethodCommand' + } + + It "Invoke-RestMethod -Resume Downloads the whole file when the file does not exist" { + # ensure the file does not exist + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + + Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-RestMethod -Resume overwrites an existing file that is larger than the remote file" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'false' + $Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-RestMethod -Resume overwrites existing file when remote server does not support resume" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-RestMethod -Resume resumes downloading from bytes" -TestCases @( + @{bytes = 4} + @{bytes = 8} + @{bytes = 12} + @{bytes = 16} + ) { + param($bytes) + # Simulate partial download + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue "Bytes/$bytes" + $null = Invoke-RestMethod -uri $uri -OutFile $outFile + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes + + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-RestMethod -Resume assumes the file was successfully completed when the local and remote file are the same size." { + # Download the entire file + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $null = Invoke-RestMethod -uri $uri -OutFile $outFile + $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + $Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" + } + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy diff --git a/test/tools/Modules/WebListener/WebListener.psm1 b/test/tools/Modules/WebListener/WebListener.psm1 index 95856f0e20..c477067ee2 100644 --- a/test/tools/Modules/WebListener/WebListener.psm1 +++ b/test/tools/Modules/WebListener/WebListener.psm1 @@ -161,6 +161,7 @@ function Get-WebListenerUrl { 'Redirect', 'Response', 'ResponseHeaders', + 'Resume', '/' )] [String]$Test, diff --git a/test/tools/WebListener/Constants.cs b/test/tools/WebListener/Constants.cs index ea4276318d..db4585163a 100644 --- a/test/tools/WebListener/Constants.cs +++ b/test/tools/WebListener/Constants.cs @@ -14,9 +14,4 @@ namespace mvc.Controllers public const string NoUrlLinkHeader = "<>; rel=\"next\""; } - - internal static class StatusCodes - { - public const Int32 ApplicationError = 500; - } } diff --git a/test/tools/WebListener/Controllers/ResponseController.cs b/test/tools/WebListener/Controllers/ResponseController.cs index 40bf06a2cc..23c0a461ff 100644 --- a/test/tools/WebListener/Controllers/ResponseController.cs +++ b/test/tools/WebListener/Controllers/ResponseController.cs @@ -74,7 +74,7 @@ namespace mvc.Controllers catch (Exception ex) { output = JsonConvert.SerializeObject(ex); - Response.StatusCode = StatusCodes.ApplicationError; + Response.StatusCode = StatusCodes.Status500InternalServerError; contentType = Constants.ApplicationJson; } } diff --git a/test/tools/WebListener/Controllers/ResumeController.cs b/test/tools/WebListener/Controllers/ResumeController.cs new file mode 100644 index 0000000000..3d0de50bd9 --- /dev/null +++ b/test/tools/WebListener/Controllers/ResumeController.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Net.Http.Headers; +using System.Net.Mime; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; +using mvc.Models; + +using RangeItemHeaderValue = System.Net.Http.Headers.RangeItemHeaderValue; +using RangeHeaderValue = System.Net.Http.Headers.RangeHeaderValue; + +namespace mvc.Controllers +{ + public class ResumeController : Controller + { + private static Byte[] FileBytes = new Byte[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}; + + public async void Index() + { + SetResumeResponseHeaders(); + string rangeHeader; + int from = 0; + int to = FileBytes.Length - 1; + if (TryGetRangeHeader(out rangeHeader)) + { + var range = GetRange(rangeHeader); + if(range.From != null) + { + from = (int)range.From; + } + if(range.To != null) + { + to = (int)range.To; + } + + } + else + { + Response.ContentType = MediaTypeNames.Application.Octet; + Response.StatusCode = StatusCodes.Status200OK; + await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); + return; + } + + if(to >= FileBytes.Length || from >= FileBytes.Length) + { + Response.StatusCode = StatusCodes.Status416RequestedRangeNotSatisfiable; + Response.Headers[HeaderNames.ContentRange] = $"bytes */{FileBytes.Length}"; + return; + } + else + { + Response.ContentType = MediaTypeNames.Application.Octet; + Response.ContentLength = to - from + 1; + Response.Headers[HeaderNames.ContentRange] = $"bytes {from}-{to}/{FileBytes.Length}"; + Response.StatusCode = StatusCodes.Status206PartialContent; + await Response.Body.WriteAsync(FileBytes, from, (int)Response.ContentLength); + } + } + + public async void NoResume() + { + SetResumeResponseHeaders(); + Response.ContentType = MediaTypeNames.Application.Octet; + Response.ContentLength = FileBytes.Length; + Response.StatusCode = StatusCodes.Status200OK; + await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); + } + + public async void Bytes(int NumberBytes) + { + if (NumberBytes > FileBytes.Length || NumberBytes < 0) + { + NumberBytes = FileBytes.Length; + } + Response.ContentType = MediaTypeNames.Application.Octet; + Response.ContentLength = NumberBytes; + await Response.Body.WriteAsync(FileBytes, 0, NumberBytes); + } + + public IActionResult Error() + { + return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); + } + + private RangeItemHeaderValue GetRange(string rangeHeader) + { + return RangeHeaderValue.Parse(rangeHeader).Ranges.FirstOrDefault(); + } + + private void SetResumeResponseHeaders() + { + string rangeHeader; + if (TryGetRangeHeader(out rangeHeader)) + { + Response.Headers["X-WebListener-Has-Range"] = "true"; + Response.Headers["X-WebListener-Request-Range"] = rangeHeader; + } + else + { + Response.Headers["X-WebListener-Has-Range"] = "false"; + } + } + + private bool TryGetRangeHeader(out string rangeHeader) + { + var rangeHeaderSv = new StringValues(); + if(Request.Headers.TryGetValue("Range", out rangeHeaderSv)) + { + rangeHeader = rangeHeaderSv.FirstOrDefault(); + return true; + } + else + { + rangeHeader = string.Empty; + return false; + } + } + } +} diff --git a/test/tools/WebListener/README.md b/test/tools/WebListener/README.md index bc5cfc98e3..1e961542ed 100644 --- a/test/tools/WebListener/README.md +++ b/test/tools/WebListener/README.md @@ -613,3 +613,72 @@ Body: "x-header-01": "value01" } ``` + +### /Resume/ + +This endpoint simulates the download of a 20 byte file with support for resuming with the use of the `Range` HTTP request header. +The bytes returned are numbered 1 to 20 inclusive. +If the `Range` header is greater than 20, the endpoint will return a `416 Requested Range Not Satisfiable` response. +The endpoint also returns an `X-WebListener-Has-Range` response header containing `true` or `false` if the HTTP Request contains a `Range` request header. +The endpoint will also return an `X-WebListener-Request-Range` response header which contains the `Range` header value if one was present. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' +$response = Invoke-WebRequest -Uri $uri -Headers @{"Range" = "bytes=0-"} +``` + +Response Headers: + +```none +HTTP/1.1 206 PartialContent +Date: Tue, 20 Mar 2018 08:45:42 GMT +Server: Kestrel +X-WebListener-Has-Range: true +X-WebListener-Request-Range: bytes=0- +Content-Length: 20 +Content-Type: application/octet-stream +Content-Range: bytes 0-19/20 +``` + +### /Resume/Bytes/{NumberBytes} + +This endpoint emulates a partial download of the same 20 bytes provided by the `/Resume/` endpoint. +The endpoint will return `{NumberBytes}` bytes of the 20 bytes. +For example `/Resume/Bytes/5` will return bytes 1 through 5 inclusive of the 20 byte file. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'Bytes/5' +$response = Invoke-WebRequest -Uri $uri +``` + +Response Headers: + +```none +HTTP/1.1 200 OK +Date: Tue, 20 Mar 2018 08:50:57 GMT +Server: Kestrel +Content-Length: 5 +Content-Type: application/octet-stream +``` + +### /Resume/NoResume + +This endpoint is the same as `/Resume/` with the exception that it ignores the `Range` HTTP request header. +This endpoint always returns the full 20 bytes and a `200` status. +The `X-WebListener-Has-Range` and `X-WebListener-Request-Range` headers are also returned the same as the `/Resume/` endpoint. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' +$response = Invoke-WebRequest -Uri $uri +``` + +Response Headers: + +```none +HTTP/1.1 200 OK +Date: Tue, 20 Mar 2018 08:48:21 GMT +Server: Kestrel +X-WebListener-Has-Range: false +Content-Length: 20 +Content-Type: application/octet-stream +``` diff --git a/test/tools/WebListener/Startup.cs b/test/tools/WebListener/Startup.cs index 9cf4844d57..775357d181 100644 --- a/test/tools/WebListener/Startup.cs +++ b/test/tools/WebListener/Startup.cs @@ -44,6 +44,10 @@ namespace mvc app.UseMvc(routes => { + routes.MapRoute( + name: "resume_bytes", + template: "Resume/Bytes/{NumberBytes?}", + defaults: new {controller = "Resume", action = "Bytes"}); routes.MapRoute( name: "default", template: "{controller=Home}/{action=Index}/{id?}");