admin/PowerShell-PowerShell
1
0
Fork 0
mirror of https://github.com/PowerShell/PowerShell synced 2026-06-08 12:12:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve the performance of 'Group-Object' (#7410)

Browse Source
This commit is contained in:
Staffan Gustafsson
2018-08-09 12:11:06 -07:00
committed by Dongbo Wang
parent 618d9f31a0
commit 037e12eddc
2 changed files with 258 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Microsoft.PowerShell.Commands.Utility/commands/utility/group-object.cs
+257 -92
View File
@@ -6,8 +6,10 @@ using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Collections.Specialized;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Management.Automation;
using System.Management.Automation.Internal;
using System.Text;
@@ -22,28 +24,27 @@ namespace Microsoft.PowerShell.Commands
/// <summary>
/// ArrayToTuple is a helper method used to create a tuple for the supplied input array.
/// </summary>
/// <typeparam name="T">The first generic type parameter.</typeparam>
/// <param name="inputObjects">Input objects used to create a tuple.</param>
/// <returns>Tuple object.</returns>
internal static object ArrayToTuple(object[] inputObjects)
internal static object ArrayToTuple<T>(IList<T> inputObjects)
{
Diagnostics.Assert(inputObjects != null, "inputObjects is null");
Diagnostics.Assert(inputObjects.Length > 0, "inputObjects is empty");
return ArrayToTuple(inputObjects, 0);
}
/// <summary>
/// ArrayToTuple is a helper method used to create a tuple for the supplied input array.
/// </summary>
/// <param name="inputObjects">Input objects used to create a tuple</param>
/// <typeparam name="T">The first generic type parameter.</typeparam>
/// <param name="inputObjects">Input objects used to create a tuple.</param>
/// <param name="startIndex">Start index of the array from which the objects have to considered for the tuple creation.</param>
/// <returns>Tuple object.</returns>
internal static object ArrayToTuple(object[] inputObjects, int startIndex)
private static object ArrayToTuple<T>(IList<T> inputObjects, int startIndex)
{
Diagnostics.Assert(inputObjects != null, "inputObjects is null");
Diagnostics.Assert(inputObjects.Length > 0, "inputObjects is empty");
Diagnostics.Assert(inputObjects.Count > 0, "inputObjects is empty");
switch (inputObjects.Length - startIndex)
switch (inputObjects.Count - startIndex)
{
case 0:
return null;
@@ -56,19 +57,49 @@ namespace Microsoft.PowerShell.Commands
case 4:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3]);
case 5:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3], inputObjects[startIndex + 4]);
return Tuple.Create(
inputObjects[startIndex],
inputObjects[startIndex + 1],
inputObjects[startIndex + 2],
inputObjects[startIndex + 3],
inputObjects[startIndex + 4]);
case 6:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3], inputObjects[startIndex + 4],
return Tuple.Create(
inputObjects[startIndex],
inputObjects[startIndex + 1],
inputObjects[startIndex + 2],
inputObjects[startIndex + 3],
inputObjects[startIndex + 4],
inputObjects[startIndex + 5]);
case 7:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3], inputObjects[startIndex + 4],
inputObjects[startIndex + 5], inputObjects[startIndex + 6]);
return Tuple.Create(
inputObjects[startIndex],
inputObjects[startIndex + 1],
inputObjects[startIndex + 2],
inputObjects[startIndex + 3],
inputObjects[startIndex + 4],
inputObjects[startIndex + 5],
inputObjects[startIndex + 6]);
case 8:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3], inputObjects[startIndex + 4],
inputObjects[startIndex + 5], inputObjects[startIndex + 6], inputObjects[startIndex + 7]);
return Tuple.Create(
inputObjects[startIndex],
inputObjects[startIndex + 1],
inputObjects[startIndex + 2],
inputObjects[startIndex + 3],
inputObjects[startIndex + 4],
inputObjects[startIndex + 5],
inputObjects[startIndex + 6],
inputObjects[startIndex + 7]);
default:
return Tuple.Create(inputObjects[startIndex], inputObjects[startIndex + 1], inputObjects[startIndex + 2], inputObjects[startIndex + 3], inputObjects[startIndex + 4],
inputObjects[startIndex + 5], inputObjects[startIndex + 6], ArrayToTuple(inputObjects, startIndex + 7));
return Tuple.Create(
inputObjects[startIndex],
inputObjects[startIndex + 1],
inputObjects[startIndex + 2],
inputObjects[startIndex + 3],
inputObjects[startIndex + 4],
inputObjects[startIndex + 5],
inputObjects[startIndex + 6],
ArrayToTuple(inputObjects, startIndex + 7));
}
}
}
@@ -78,8 +109,7 @@ namespace Microsoft.PowerShell.Commands
/// </summary>
public sealed class GroupInfoNoElement : GroupInfo
{
internal GroupInfoNoElement(OrderByPropertyEntry groupValue)
: base(groupValue)
internal GroupInfoNoElement(OrderByPropertyEntry groupValue) : base(groupValue)
{
}
@@ -92,6 +122,7 @@ namespace Microsoft.PowerShell.Commands
/// <summary>
/// Emitted by Group-Object.
/// </summary>
[DebuggerDisplay("{Name} ({Count})")]
public class GroupInfo
{
internal GroupInfo(OrderByPropertyEntry groupValue)
@@ -113,17 +144,17 @@ namespace Microsoft.PowerShell.Commands
StringBuilder sb = new StringBuilder();
foreach (ObjectCommandPropertyValue propValue in propValues)
{
if (propValue != null && propValue.PropertyValue != null)
var propValuePropertyValue = propValue?.PropertyValue;
if (propValuePropertyValue != null)
{
var propertyValueItems = propValue.PropertyValue as ICollection;
if (propertyValueItems != null)
if (propValuePropertyValue is ICollection propertyValueItems)
{
sb.Append("{");
var length = sb.Length;
foreach (object item in propertyValueItems)
{
sb.Append(string.Format(CultureInfo.InvariantCulture, "{0}, ", item.ToString()));
sb.AppendFormat(CultureInfo.InvariantCulture, "{0}, ", item.ToString());
}
sb = sb.Length > length ? sb.Remove(sb.Length - 2, 2) : sb;
@@ -131,15 +162,16 @@ namespace Microsoft.PowerShell.Commands
}
else
{
sb.Append(string.Format(CultureInfo.InvariantCulture, "{0}, ", propValue.PropertyValue.ToString()));
sb.AppendFormat(CultureInfo.InvariantCulture, "{0}, ", propValuePropertyValue.ToString());
}
}
}
return sb.Length >= 2 ? sb.Remove(sb.Length - 2, 2).ToString() : string.Empty;
}
/// <summary>
/// Values of the group.
/// Gets the values of the group.
/// </summary>
public ArrayList Values
{
@@ -150,29 +182,30 @@ namespace Microsoft.PowerShell.Commands
{
values.Add(propValue.PropertyValue);
}
return values;
}
}
/// <summary>
/// Number of objects in the group.
/// Gets the number of objects in the group.
/// </summary>
public int Count { get; internal set; }
/// <summary>
/// The list of objects in this group.
/// Gets the list of objects in this group.
/// </summary>
public Collection<PSObject> Group { get; } = null;
public Collection<PSObject> Group { get; }
/// <summary>
/// The name of the group.
/// Gets the name of the group.
/// </summary>
public string Name { get; } = null;
public string Name { get; }
/// <summary>
/// The OrderByPropertyEntry used to build this group object.
/// Gets the OrderByPropertyEntry used to build this group object.
/// </summary>
internal OrderByPropertyEntry GroupValue { get; } = null;
internal OrderByPropertyEntry GroupValue { get; }
}
/// <summary>
@@ -187,30 +220,22 @@ namespace Microsoft.PowerShell.Commands
/// <summary>
/// An instance of the PSTraceSource class used for trace output.
/// </summary>
[TraceSourceAttribute(
"GroupObjectCommand",
"Class that has group base implementation")]
private static PSTraceSource s_tracer =
PSTraceSource.GetTracer("GroupObjectCommand",
"Class that has group base implementation");
[TraceSource("GroupObjectCommand", "Class that has group base implementation")]
private static readonly PSTraceSource s_tracer = PSTraceSource.GetTracer("GroupObjectCommand", "Class that has group base implementation");
#endregion tracer
#region Command Line Switches
/// <summary>
/// Flatten the groups.
/// Gets or sets the NoElement parameter indicating of the groups should be flattened.
/// </summary>
/// <value></value>
[Parameter]
public SwitchParameter NoElement
{
get { return _noElement; }
set { _noElement = value; }
}
private bool _noElement;
public SwitchParameter NoElement { get; set; }
/// <summary>
/// the AsHashTable parameter.
/// Gets or sets the AsHashTable parameter.
/// </summary>
/// <value></value>
[Parameter(ParameterSetName = "HashTable")]
@@ -219,16 +244,19 @@ namespace Microsoft.PowerShell.Commands
public SwitchParameter AsHashTable { get; set; }
/// <summary>
/// Gets or sets the AsString parameter.
/// </summary>
/// <value></value>
[Parameter(ParameterSetName = "HashTable")]
public SwitchParameter AsString { get; set; }
private List<GroupInfo> _groups = new List<GroupInfo>();
private OrderByProperty _orderByProperty = new OrderByProperty();
private readonly List<GroupInfo> _groups = new List<GroupInfo>();
private readonly OrderByProperty _orderByProperty = new OrderByProperty();
private readonly Dictionary<object, GroupInfo> _tupleToGroupInfoMappingDictionary = new Dictionary<object, GroupInfo>();
private readonly List<OrderByPropertyEntry> _entriesToOrder = new List<OrderByPropertyEntry>();
private OrderByPropertyComparer _orderByPropertyComparer;
private bool _hasProcessedFirstInputObject;
private Dictionary<object, GroupInfo> _tupleToGroupInfoMappingDictionary = new Dictionary<object, GroupInfo>();
private OrderByPropertyComparer _orderByPropertyComparer = null;
private bool _hasDifferentValueTypes;
private Type[] _propertyTypesCandidate;
#endregion
@@ -238,25 +266,26 @@ namespace Microsoft.PowerShell.Commands
/// Utility function called by Group-Object to create Groups.
/// </summary>
/// <param name="currentObjectEntry">Input object that needs to be grouped.</param>
/// <param name="noElement">true if we are not accumulating objects</param>
/// <param name="noElement">True if we are not accumulating objects.</param>
/// <param name="groups">List containing Groups.</param>
/// <param name="groupInfoDictionary">Dictionary used to keep track of the groups with hash of the property values being the key.</param>
/// <param name="orderByPropertyComparer">The Comparer to be used while comparing to check if new group has to be created.</param>
internal static void DoGrouping(OrderByPropertyEntry currentObjectEntry, bool noElement, List<GroupInfo> groups, Dictionary<object, GroupInfo> groupInfoDictionary,
private static void DoGrouping(
OrderByPropertyEntry currentObjectEntry,
bool noElement,
List<GroupInfo> groups,
Dictionary<object, GroupInfo> groupInfoDictionary,
OrderByPropertyComparer orderByPropertyComparer)
{
if (currentObjectEntry != null && currentObjectEntry.orderValues != null && currentObjectEntry.orderValues.Count > 0)
var currentObjectOrderValues = currentObjectEntry.orderValues;
if (currentObjectOrderValues != null && currentObjectOrderValues.Count > 0)
{
object currentTupleObject = PSTuple.ArrayToTuple(currentObjectEntry.orderValues.ToArray());
object currentTupleObject = PSTuple.ArrayToTuple(currentObjectOrderValues);
GroupInfo currentGroupInfo = null;
if (groupInfoDictionary.TryGetValue(currentTupleObject, out currentGroupInfo))
if (groupInfoDictionary.TryGetValue(currentTupleObject, out var currentGroupInfo))
{
if (currentGroupInfo != null)
{
//add this inputObject to an existing group
currentGroupInfo.Add(currentObjectEntry.inputObject);
}
// add this inputObject to an existing group
currentGroupInfo.Add(currentObjectEntry.inputObject);
}
else
{
@@ -277,7 +306,7 @@ namespace Microsoft.PowerShell.Commands
if (!isCurrentItemGrouped)
{
// create a new group
s_tracer.WriteLine("Create a new group: {0}", currentObjectEntry.orderValues);
s_tracer.WriteLine("Create a new group: {0}", currentObjectOrderValues);
GroupInfo newObjGrp = noElement ? new GroupInfoNoElement(currentObjectEntry) : new GroupInfo(currentObjectEntry);
groups.Add(newObjGrp);
@@ -286,12 +315,71 @@ namespace Microsoft.PowerShell.Commands
}
}
}
private void WriteNonTerminatingError(Exception exception, string resourceIdAndErrorId,
ErrorCategory category)
/// <summary>
/// Utility function called by Group-Object to create Groups.
/// </summary>
/// <param name="currentObjectEntry">Input object that needs to be grouped.</param>
/// <param name="noElement">True if we are not accumulating objects.</param>
/// <param name="groups">List containing Groups.</param>
/// <param name="groupInfoDictionary">Dictionary used to keep track of the groups with hash of the property values being the key.</param>
/// <param name="orderByPropertyComparer">The Comparer to be used while comparing to check if new group has to be created.</param>
private static void DoOrderedGrouping(
OrderByPropertyEntry currentObjectEntry,
bool noElement,
List<GroupInfo> groups,
Dictionary<object, GroupInfo> groupInfoDictionary,
OrderByPropertyComparer orderByPropertyComparer)
{
var currentObjectOrderValues = currentObjectEntry.orderValues;
if (currentObjectOrderValues != null && currentObjectOrderValues.Count > 0)
{
object currentTupleObject = PSTuple.ArrayToTuple(currentObjectOrderValues);
if (groupInfoDictionary.TryGetValue(currentTupleObject, out var currentGroupInfo))
{
// add this inputObject to an existing group
currentGroupInfo.Add(currentObjectEntry.inputObject);
}
else
{
bool isCurrentItemGrouped = false;
if (groups.Count > 0)
{
var lastGroup = groups[groups.Count - 1];
// Check if the current input object can be converted to one of the already known types
// by looking up in the type to GroupInfo mapping.
if (orderByPropertyComparer.Compare(lastGroup.GroupValue, currentObjectEntry) == 0)
{
lastGroup.Add(currentObjectEntry.inputObject);
isCurrentItemGrouped = true;
}
}
if (!isCurrentItemGrouped)
{
// create a new group
s_tracer.WriteLine("Create a new group: {0}", currentObjectOrderValues);
GroupInfo newObjGrp = noElement
? new GroupInfoNoElement(currentObjectEntry)
: new GroupInfo(currentObjectEntry);
groups.Add(newObjGrp);
groupInfoDictionary.Add(currentTupleObject, newObjGrp);
}
}
}
}
private void WriteNonTerminatingError(Exception exception, string resourceIdAndErrorId, ErrorCategory category)
{
Exception ex = new Exception(StringUtil.Format(resourceIdAndErrorId), exception);
WriteError(new ErrorRecord(ex, resourceIdAndErrorId, category, null));
}
#endregion utils
/// <summary>
@@ -301,7 +389,7 @@ namespace Microsoft.PowerShell.Commands
{
if (InputObject != null && InputObject != AutomationNull.Value)
{
OrderByPropertyEntry currentEntry = null;
OrderByPropertyEntry currentEntry;
if (!_hasProcessedFirstInputObject)
{
@@ -309,14 +397,30 @@ namespace Microsoft.PowerShell.Commands
{
Property = OrderByProperty.GetDefaultKeyPropertySet(InputObject);
}
_orderByProperty.ProcessExpressionParameter(this, Property);
if (AsString && !AsHashTable)
{
ArgumentException ex = new ArgumentException(UtilityCommonStrings.GroupObjectWithHashTable);
ErrorRecord er = new ErrorRecord(ex, "ArgumentException", ErrorCategory.InvalidArgument, AsString);
ThrowTerminatingError(er);
}
if (AsHashTable && !AsString && (Property != null && (Property.Length > 1 || _orderByProperty.MshParameterList.Count > 1)))
{
ArgumentException ex = new ArgumentException(UtilityCommonStrings.GroupObjectSingleProperty);
ErrorRecord er = new ErrorRecord(ex, "ArgumentException", ErrorCategory.InvalidArgument, Property);
ThrowTerminatingError(er);
}
currentEntry = _orderByProperty.CreateOrderByPropertyEntry(this, InputObject, CaseSensitive, _cultureInfo);
bool[] ascending = new bool[currentEntry.orderValues.Count];
for (int index = 0; index < currentEntry.orderValues.Count; index++)
{
ascending[index] = true;
}
_orderByPropertyComparer = new OrderByPropertyComparer(ascending, _cultureInfo, CaseSensitive);
_hasProcessedFirstInputObject = true;
@@ -326,40 +430,109 @@ namespace Microsoft.PowerShell.Commands
currentEntry = _orderByProperty.CreateOrderByPropertyEntry(this, InputObject, CaseSensitive, _cultureInfo);
}
DoGrouping(currentEntry, this.NoElement, _groups, _tupleToGroupInfoMappingDictionary, _orderByPropertyComparer);
_entriesToOrder.Add(currentEntry);
var currentEntryOrderValues = currentEntry.orderValues;
if (!_hasDifferentValueTypes)
{
UpdateOrderPropertyTypeInfo(currentEntryOrderValues);
}
}
}
private void UpdateOrderPropertyTypeInfo(List<ObjectCommandPropertyValue> currentEntryOrderValues)
{
if (_propertyTypesCandidate == null)
{
_propertyTypesCandidate = currentEntryOrderValues.Select(c => PSObject.Base(c.PropertyValue)?.GetType()).ToArray();
return;
}
if (_propertyTypesCandidate.Length != currentEntryOrderValues.Count)
{
_hasDifferentValueTypes = true;
return;
}
// check all the types we group on.
// if we find more than one set of types, _hasDifferentValueTypes is set to true,
// and we are forced to take a slower code path when we group our objects
for (int i = 0; i < _propertyTypesCandidate.Length; i++)
{
var candidateType = _propertyTypesCandidate[i];
var propertyType = PSObject.Base(currentEntryOrderValues[i].PropertyValue)?.GetType();
if (propertyType == null)
{
// we ignore properties without values. We can always compare against null.
continue;
}
// if we haven't gotten a type for a property yet, update it when we do get a value
if (propertyType != candidateType)
{
if (candidateType == null)
{
_propertyTypesCandidate[i] = propertyType;
}
else
{
_hasDifferentValueTypes = true;
break;
}
}
}
}
/// <summary>
/// Completes the processing of the gathered group objects.
/// </summary>
protected override void EndProcessing()
{
if (!_hasDifferentValueTypes)
{
// using OrderBy to get stable sort.
// fast path when we only have the same object types to group
foreach (var entry in _entriesToOrder.OrderBy(e => e, _orderByPropertyComparer))
{
DoOrderedGrouping(entry, NoElement, _groups, _tupleToGroupInfoMappingDictionary, _orderByPropertyComparer);
if (Stopping)
{
return;
}
}
}
else
{
foreach (var entry in _entriesToOrder)
{
DoGrouping(entry, NoElement, _groups, _tupleToGroupInfoMappingDictionary, _orderByPropertyComparer);
if (Stopping)
{
return;
}
}
}
s_tracer.WriteLine(_groups.Count);
if (_groups.Count > 0)
{
if (AsHashTable)
{
Hashtable _table = CollectionsUtil.CreateCaseInsensitiveHashtable();
Hashtable hashtable = CollectionsUtil.CreateCaseInsensitiveHashtable();
try
{
foreach (GroupInfo _grp in _groups)
if (AsString)
{
if (AsString)
foreach (GroupInfo grp in _groups)
{
_table.Add(_grp.Name, _grp.Group);
hashtable.Add(grp.Name, grp.Group);
}
else
}
else
{
foreach (GroupInfo grp in _groups)
{
if (_grp.Values.Count == 1)
{
_table.Add(PSObject.Base(_grp.Values[0]), _grp.Group);
}
else
{
ArgumentException ex = new ArgumentException(UtilityCommonStrings.GroupObjectSingleProperty);
ErrorRecord er = new ErrorRecord(ex, "ArgumentException", ErrorCategory.InvalidArgument, Property);
ThrowTerminatingError(er);
}
hashtable.Add(PSObject.Base(grp.Values[0]), grp.Group);
}
}
}
@@ -368,20 +541,12 @@ namespace Microsoft.PowerShell.Commands
WriteNonTerminatingError(e, UtilityCommonStrings.InvalidOperation, ErrorCategory.InvalidArgument);
return;
}
WriteObject(_table);
WriteObject(hashtable);
}
else
{
if (AsString)
{
ArgumentException ex = new ArgumentException(UtilityCommonStrings.GroupObjectWithHashTable);
ErrorRecord er = new ErrorRecord(ex, "ArgumentException", ErrorCategory.InvalidArgument, AsString);
ThrowTerminatingError(er);
}
else
{
WriteObject(_groups, true);
}
WriteObject(_groups, true);
}
}
}
src/System.Management.Automation/engine/CommandBase.cs
+1 -1
View File
@@ -35,7 +35,7 @@ namespace System.Management.Automation.Internal
/// Ideally this would be an internal class, but C# does not support
/// public classes deriving from internal classes.
/// -->
[DebuggerDisplay("Command = {commandInfo}")]
[DebuggerDisplay("Command = {_commandInfo}")]
public abstract class InternalCommand
{
#region private_members