admin/PowerShell-PowerShell
1
0
Fork 0
mirror of https://github.com/PowerShell/PowerShell synced 2026-06-08 12:12:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support Link Header pagination in WebCmdlets (#3828)

Browse Source 
* Support Link Header pagination in WebCmdlets to make it easier for the end user implementing:
https://github.com/PowerShell/PowerShell-RFC/blob/master/2-Draft-Accepted/RFC0021-Link-header-based-pagination-for-WebCmdlets.md

When the response includes a Link Header (https://tools.ietf.org/html/rfc5988#page-6), for Invoke-WebRequest we
create a RelationLink property that is a Dictionary representing the URLs and rel attributes and ensure the
URLs are absolute to make it easier for the developer to use.  For Invoke-RestMethod, we expose a -FollowRelLink
switch to automatically follow 'next' rel links to the end until we hit the optional -MaxRelLink parameter value.

* removed unnecessary refs to namespaces

* addressed code review feedback
This commit is contained in:
Steve Lee
2017-05-24 15:54:02 -07:00
committed by Travis Plunk
parent 99f9ef22d2
commit 40446c80a7
8 changed files with 337 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs
+23
View File
@@ -37,6 +37,29 @@ namespace Microsoft.PowerShell.Commands
set { base.CustomMethod = value; }
}
/// <summary>
/// enable automatic following of rel links
/// </summary>
[Parameter]
[Alias("FL")]
public SwitchParameter FollowRelLink
{
get { return base._followRelLink; }
set { base._followRelLink = value; }
}
/// <summary>
/// gets or sets the maximum number of rel links to follow
/// </summary>
[Parameter]
[Alias("ML")]
[ValidateRange(1, Int32.MaxValue)]
public int MaximumFollowRelLink
{
get { return base._maximumFollowRelLink; }
set { base._maximumFollowRelLink = value; }
}
#endregion Parameters
#region Helper Methods
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs
+9
View File
@@ -20,6 +20,14 @@ namespace Microsoft.PowerShell.Commands
{
#region Virtual Method Overrides
/// <summary>
/// Default constructor for InvokeWebRequestCommand
/// </summary>
public InvokeWebRequestCommand() : base()
{
this._parseRelLink = true;
}
/// <summary>
/// Process the web response and output corresponding objects.
/// </summary>
@@ -46,6 +54,7 @@ namespace Microsoft.PowerShell.Commands
// creating a MemoryStream wrapper to response stream here to support IsStopping.
responseStream = new WebResponseContentMemoryStream(responseStream, StreamHelper.ChunkSize, this);
WebResponseObject ro = WebResponseObjectFactory.GetResponseObject(response, responseStream, this.Context, UseBasicParsing);
ro.RelationLink = _relationLink;
WriteObject(ro);
// use the rawcontent stream from WebResponseObject for further
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs
+158 -71
View File
@@ -16,6 +16,9 @@ using System.Globalization;
using System.Security.Cryptography;
using System.Threading;
using System.Xml;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Linq;
namespace Microsoft.PowerShell.Commands
{
@@ -61,6 +64,26 @@ namespace Microsoft.PowerShell.Commands
/// </summary>
private CancellationTokenSource _cancelToken = null;
/// <summary>
/// Parse Rel Links
/// </summary>
internal bool _parseRelLink = false;
/// <summary>
/// Automatically follow Rel Links
/// </summary>
internal bool _followRelLink = false;
/// <summary>
/// Automatically follow Rel Links
/// </summary>
internal Dictionary<string, string> _relationLink = null;
/// <summary>
/// Maximum number of Rel Links to follow
/// </summary>
internal int _maximumFollowRelLink = Int32.MaxValue;
private HttpMethod GetHttpMethod(WebRequestMethod method)
{
switch (Method)
@@ -234,7 +257,7 @@ namespace Microsoft.PowerShell.Commands
}
// Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent
// Default behaviour is continue to send body content anyway after a short period
// Default behavior is continue to send body content anyway after a short period
// Here it send the two part as a whole.
request.Headers.ExpectContinue = false;
@@ -374,91 +397,121 @@ namespace Microsoft.PowerShell.Commands
PrepareSession();
using (HttpClient client = GetHttpClient())
using (HttpRequestMessage request = GetRequest(Uri))
{
FillRequestStream(request);
try
int followedRelLink = 0;
Uri uri = Uri;
do
{
long requestContentLength = 0;
if (request.Content != null)
requestContentLength = request.Content.Headers.ContentLength.Value;
string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
"{0} {1} with {2}-byte payload",
request.Method,
request.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);
HttpResponseMessage response = GetResponse(client, request);
string contentType = ContentHelper.GetContentType(response);
string respVerboseMsg = string.Format(CultureInfo.CurrentCulture,
"received {0}-byte response of content type {1}",
response.Content.Headers.ContentLength,
contentType);
WriteVerbose(respVerboseMsg);
if (!response.IsSuccessStatusCode)
if (followedRelLink > 0)
{
string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure,
(int)response.StatusCode, response.ReasonPhrase);
HttpResponseException httpEx = new HttpResponseException(message, response);
ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
string detailMsg = "";
StreamReader reader = null;
string linkVerboseMsg = string.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.FollowingRelLinkVerboseMsg,
uri.AbsoluteUri);
WriteVerbose(linkVerboseMsg);
}
using (HttpRequestMessage request = GetRequest(uri))
{
FillRequestStream(request);
try
{
reader = new StreamReader(StreamHelper.GetResponseStream(response));
// remove HTML tags making it easier to read
detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>","");
}
catch (Exception)
{
// catch all
}
finally
{
if (reader != null)
long requestContentLength = 0;
if (request.Content != null)
requestContentLength = request.Content.Headers.ContentLength.Value;
string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.WebMethodInvocationVerboseMsg,
request.Method,
request.RequestUri,
requestContentLength);
WriteVerbose(reqVerboseMsg);
HttpResponseMessage response = GetResponse(client, request);
string contentType = ContentHelper.GetContentType(response);
string respVerboseMsg = string.Format(CultureInfo.CurrentCulture,
WebCmdletStrings.WebResponseVerboseMsg,
response.Content.Headers.ContentLength,
contentType);
WriteVerbose(respVerboseMsg);
if (!response.IsSuccessStatusCode)
{
reader.Dispose();
string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure,
(int)response.StatusCode, response.ReasonPhrase);
HttpResponseException httpEx = new HttpResponseException(message, response);
ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
string detailMsg = "";
StreamReader reader = null;
try
{
reader = new StreamReader(StreamHelper.GetResponseStream(response));
// remove HTML tags making it easier to read
detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>","");
}
catch (Exception)
{
// catch all
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
if (!String.IsNullOrEmpty(detailMsg))
{
er.ErrorDetails = new ErrorDetails(detailMsg);
}
ThrowTerminatingError(er);
}
if (_parseRelLink || _followRelLink)
{
ParseLinkHeader(response, uri);
}
ProcessResponse(response);
UpdateSession(response);
// If we hit our maximum redirection count, generate an error.
// Errors with redirection counts of greater than 0 are handled automatically by .NET, but are
// impossible to detect programmatically when we hit this limit. By handling this ourselves
// (and still writing out the result), users can debug actual HTTP redirect problems.
if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false"
{
if (response.StatusCode == HttpStatusCode.Found ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently)
{
ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request);
er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded);
WriteError(er);
}
}
}
if (!String.IsNullOrEmpty(detailMsg))
catch (HttpRequestException ex)
{
er.ErrorDetails = new ErrorDetails(detailMsg);
ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
if (ex.InnerException != null)
{
er.ErrorDetails = new ErrorDetails(ex.InnerException.Message);
}
ThrowTerminatingError(er);
}
ThrowTerminatingError(er);
}
ProcessResponse(response);
UpdateSession(response);
// If we hit our maximum redirection count, generate an error.
// Errors with redirection counts of greater than 0 are handled automatically by .NET, but are
// impossible to detect programmatically when we hit this limit. By handling this ourselves
// (and still writing out the result), users can debug actual HTTP redirect problems.
if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false"
{
if (response.StatusCode == HttpStatusCode.Found ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently)
if (_followRelLink)
{
ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request);
er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded);
WriteError(er);
if (!_relationLink.ContainsKey("next"))
{
return;
}
uri = new Uri(_relationLink["next"]);
followedRelLink++;
}
}
}
catch (HttpRequestException ex)
{
ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request);
if (ex.InnerException != null)
{
er.ErrorDetails = new ErrorDetails(ex.InnerException.Message);
}
ThrowTerminatingError(er);
}
while (_followRelLink && (followedRelLink < _maximumFollowRelLink));
}
}
catch (CryptographicException ex)
@@ -619,6 +672,40 @@ namespace Microsoft.PowerShell.Commands
}
internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri)
{
if (_relationLink == null)
{
_relationLink = new Dictionary<string, string>();
}
else
{
_relationLink.Clear();
}
// we only support the URL in angle brackets and `rel`, other attributes are ignored
// user can still parse it themselves via the Headers property
string pattern = "<(?<url>.*?)>;\\srel=\"(?<rel>.*?)\"";
IEnumerable<string> links;
if (response.Headers.TryGetValues("Link", out links))
{
foreach(string link in links.FirstOrDefault().Split(","))
{
Match match = Regex.Match(link, pattern);
if (match.Success)
{
string url = match.Groups["url"].Value;
string rel = match.Groups["rel"].Value;
if (url != String.Empty && rel != String.Empty && !_relationLink.ContainsKey(rel))
{
Uri absoluteUri = new Uri(requestUri, url);
_relationLink.Add(rel, absoluteUri.AbsoluteUri.ToString());
}
}
}
}
}
#endregion Helper Methods
}
}
src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs
+5
View File
@@ -41,6 +41,11 @@ namespace Microsoft.PowerShell.Commands
}
}
/// <summary>
/// gets the RelationLink property
/// </summary>
public Dictionary<string, string> RelationLink { get; internal set; }
#endregion
#region Constructors
src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx
+10 -1
View File
@@ -213,4 +213,13 @@
<data name="ResponseStatusCodeFailure" xml:space="preserve">
<value>Response status code does not indicate success: {0} ({1}).</value>
</data>
</root>
<data name="FollowingRelLinkVerboseMsg" xml:space="preserve">
<value>Following rel link {0}</value>
</data>
<data name="WebMethodInvocationVerboseMsg" xml:space="preserve">
<value>{0} {1} with {2}-byte payload</value>
</data>
<data name="WebResponseVerboseMsg" xml:space="preserve">
<value>received {0}-byte response of content type {1}</value>
</data>
</root>
src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs
+2
View File
@@ -1271,6 +1271,7 @@ namespace System.Management.Automation.Runspaces
.AddItemProperty(@"Links")
.AddItemProperty(@"ParsedHtml")
.AddItemProperty(@"RawContentLength")
.AddItemProperty(@"RelationLink")
.EndEntry()
.EndList());
}
@@ -1291,6 +1292,7 @@ namespace System.Management.Automation.Runspaces
", label: "RawContent")
.AddItemProperty(@"Headers")
.AddItemProperty(@"RawContentLength")
.AddItemProperty(@"RelationLink")
.EndEntry()
.EndList());
}
test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
+85
View File
@@ -145,6 +145,14 @@ function GetTestData
Describe "Invoke-WebRequest tests" -Tags "Feature" {
BeforeAll {
$null = Start-HttpListener -AsJob
}
AfterAll {
$null = Stop-HttpListener
}
# Validate the output of Invoke-WebRequest
#
function ValidateResponse
@@ -509,6 +517,35 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
$result.Error.FullyQualifiedErrorId | Should Be "WebCmdletWebResponseException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand"
}
It "Validate Invoke-WebRequest returns empty RelationLink property if there is no Link Header" {
$command = "Invoke-WebRequest -Uri http://localhost:8080/PowerShell?test=response"
$result = ExecuteWebCommand -command $command
$result.Output.RelationLink.Count | Should Be 0
}
It "Validate Invoke-WebRequest returns valid RelationLink property with absolute uris if Link Header is present" {
$command = "Invoke-WebRequest -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=5'"
$result = ExecuteWebCommand -command $command
$result.Output.RelationLink.Count | Should BeExactly 2
$result.Output.RelationLink["next"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=5&linknumber=2"
$result.Output.RelationLink["last"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=5&linknumber=5"
}
It "Validate Invoke-WebRequest quietly ignores invalid Link Headers in RelationLink property" -TestCases @(
@{ type = "noUrl" }
@{ type = "malformed" }
@{ type = "noRel" }
) {
param($type)
$command = "Invoke-WebRequest -Uri 'http://localhost:8080/PowerShell?test=linkheader&type=$type'"
$result = ExecuteWebCommand -command $command
$result.Output.RelationLink.Count | Should BeExactly 1
$result.Output.RelationLink["last"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=3&linknumber=3"
}
BeforeEach {
if ($env:http_proxy) {
$savedHttpProxy = $env:http_proxy
@@ -538,6 +575,14 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" {
Describe "Invoke-RestMethod tests" -Tags "Feature" {
BeforeAll {
$null = Start-HttpListener -AsJob
}
AfterAll {
$null = Stop-HttpListener
}
It "Invoke-RestMethod returns User-Agent" {
$command = "Invoke-RestMethod -Uri http://httpbin.org/user-agent -TimeoutSec 5"
@@ -872,6 +917,46 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" {
$result.Error.FullyQualifiedErrorId | Should Be "WebCmdletWebResponseException,Microsoft.PowerShell.Commands.InvokeRestMethodCommand"
}
It "Validate Invoke-RestMethod -FollowRelLink doesn't fail if no Link Header is present" {
$command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=response&output=foo' -FollowRelLink"
$result = ExecuteWebCommand -command $command
$result.Output | Should BeExactly "foo"
}
It "Validate Invoke-RestMethod -FollowRelLink correctly follows all the available relation links" {
$maxLinks = 5
$command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=$maxlinks' -FollowRelLink"
$result = ExecuteWebCommand -command $command
$result.Output.output.Count | Should BeExactly $maxLinks
1..$maxLinks | ForEach-Object { $result.Output.output[$_ - 1] | Should BeExactly $_ }
}
It "Validate Invoke-RestMethod -FollowRelLink correctly limits to -MaximumRelLink" {
$maxLinks = 10
$maxLinksToFollow = 6
$command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=$maxlinks' -FollowRelLink -MaximumFollowRelLink $maxLinksToFollow"
$result = ExecuteWebCommand -command $command
$result.Output.output.Count | Should BeExactly $maxLinksToFollow
1..$maxLinksToFollow | ForEach-Object { $result.Output.output[$_ - 1] | Should BeExactly $_ }
}
It "Validate Invoke-RestMethod quietly ignores invalid Link Headers if -FollowRelLink is specified" -TestCases @(
@{ type = "noUrl" }
@{ type = "malformed" }
@{ type = "noRel" }
) {
param($type)
$command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&type=$type' -FollowRelLink"
$result = ExecuteWebCommand -command $command
$result.Output.output | Should BeExactly 1
}
BeforeEach {
if ($env:http_proxy) {
$savedHttpProxy = $env:http_proxy
test/tools/Modules/HttpListener/HttpListener.psm1
+45
View File
@@ -136,6 +136,51 @@ Function Start-HTTPListener {
$output = $request | ConvertTo-Json
}
}
"linkheader"
{
$maxLinks = $queryItems["maxlinks"]
if ($maxlinks -eq $null)
{
$maxLinks = 3
}
$linkNumber = [int]$queryItems["linknumber"]
$prev = ""
if ($linkNumber -eq 0)
{
$linkNumber = 1
}
else
{
# use $urlPrefix to ensure output is resolved to absolute uri
$prev = ", <$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$($linkNumber-1); rel=`"prev`""
}
$links = ""
if ($linkNumber -lt $maxLinks)
{
switch ($queryItems["type"])
{
"noUrl"
{
$links = "<>; rel=`"next`","
}
"malformed"
{
$links = "{url}; foo,"
}
"noRel"
{
$links = "<url>; foo=`"bar`","
}
default
{
$links = "<$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$($linkNumber+1)>; rel=`"next`", "
}
}
}
$links = "$links<$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$maxlinks>; rel=`"last`"$prev"
$outputHeader.Add("Link", $links)
$output = "{ `"output`": `"$linkNumber`"}"
}
default
{
$statusCode = [System.Net.HttpStatusCode]::NotFound