admin/PowerShell-PowerShell
1
0
Fork 0
mirror of https://github.com/PowerShell/PowerShell synced 2026-06-08 12:12:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Avoid the int[] and int[,] allocation when tokenizing line comments and matching wildcard pattern (#10009)

Browse Source 
- Reusing the same 2-dimensional integer array for processing line comments in the same Tokenizer.
- Use ArrayPool<int> in PatternPositionVisitor to avoid creating transient int[].
This commit is contained in:
Dongbo Wang
2019-06-28 08:14:41 +05:00
committed by Ilya
parent 142f5dcad5
commit 026a8f1f41
2 changed files with 73 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/System.Management.Automation/engine/parser/tokenizer.cs
+26 -9
View File
@@ -1661,6 +1661,9 @@ namespace System.Management.Automation.Language
case '\n':
UngetChar();
// Detect a line comment that disguises itself to look like the beginning of a signature block.
// This could be used to hide code at the bottom of a script, since people might assume there is nothing else after the signature.
//
// The token similarity threshold was chosen by instrumenting the tokenizer and
// analyzing every comment from PoshCode, Technet Script Center, and Windows.
//
@@ -1677,15 +1680,16 @@ namespace System.Management.Automation.Language
//
// There were only 279 (out of 269,387) comments with a similarity of 11,12,13,14, or 15.
// At a similarity of 16-77, there were thousands of comments per similarity bucket.
//
// System.IO.File.AppendAllText(@"c:\temp\signature_similarity.txt", "" + sawBeginTokenSimilarity + ":" + commentLineComparison);
const string beginSignatureTextNoSpace = "sig#beginsignatureblock\n";
const int beginTokenSimilarityThreshold = 10;
// Quick exit - the comment line is more than 'threshold' longer. Therefore,
const int beginTokenSimilarityUpperBound = 34; // beginSignatureTextNoSpace.Length + beginTokenSimilarityThreshold
const int beginTokenSimilarityLowerBound = 14; // beginSignatureTextNoSpace.Length - beginTokenSimilarityThreshold
// Quick exit - the comment line is more than 'threshold' longer, or is less than 'threshold' shorter. Therefore,
// its similarity will be over the threshold.
if (commentLine.Length > (beginSignatureTextNoSpace.Length + beginTokenSimilarityThreshold))
if (commentLine.Length > beginTokenSimilarityUpperBound || commentLine.Length < beginTokenSimilarityLowerBound)
{
sawBeginSig = false;
}
@@ -1697,10 +1701,20 @@ namespace System.Management.Automation.Language
//
// The average script is 14% comments and parses in about 5.05 ms with this algorithm,
// about 4.45 ms with the more simplistic algorithm.
//
string commentLineComparison = commentLine.ToString().ToLowerInvariant();
int sawBeginTokenSimilarity = GetStringSimilarity(commentLineComparison, beginSignatureTextNoSpace);
string commentLineComparison = commentLine.ToString().ToLowerInvariant();
if (_beginTokenSimilarity2dArray == null)
{
// Create the 2 dimensional array for edit distance calculation if it hasn't been created yet.
_beginTokenSimilarity2dArray = new int[beginTokenSimilarityUpperBound + 1, beginSignatureTextNoSpace.Length + 1];
}
else
{
// Zero out the 2 dimensional array before using it.
Array.Clear(_beginTokenSimilarity2dArray, 0, _beginTokenSimilarity2dArray.Length);
}
int sawBeginTokenSimilarity = GetStringSimilarity(commentLineComparison, beginSignatureTextNoSpace, _beginTokenSimilarity2dArray);
sawBeginSig = sawBeginTokenSimilarity < beginTokenSimilarityThreshold;
}
@@ -1717,6 +1731,9 @@ namespace System.Management.Automation.Language
#region Object reuse
// A two-dimensional integer array reused for calculating string similarity.
private int[,] _beginTokenSimilarity2dArray;
private readonly Queue<StringBuilder> _stringBuilders = new Queue<StringBuilder>();
private StringBuilder GetStringBuilder()
@@ -1796,14 +1813,14 @@ namespace System.Management.Automation.Language
// Implementation of the Levenshtein Distance algorithm
// https://en.wikipedia.org/wiki/Levenshtein_distance
private static int GetStringSimilarity(string first, string second)
private static int GetStringSimilarity(string first, string second, int[,] distanceMap = null)
{
Diagnostics.Assert(!string.IsNullOrEmpty(first) && !string.IsNullOrEmpty(second), "Caller never calls us with empty strings");
// Store a distance map to store the number of edits required to
// convert the first <row> letters of First to the first <column>
// letters of Second.
int[,] distanceMap = new int[first.Length + 1, second.Length + 1];
distanceMap ??= new int[first.Length + 1, second.Length + 1];
// Initialize the first row and column of the matrix - the number
// of edits required when one of the strings is empty is just
src/System.Management.Automation/engine/regex.cs
+47 -32
View File
@@ -3,6 +3,7 @@
#pragma warning disable 1634, 1691
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Globalization;
@@ -914,43 +915,51 @@ namespace System.Management.Automation
var patternPositionsForNextStringPosition =
new PatternPositionsVisitor(_patternElements.Length);
for (int currentStringPosition = 0;
currentStringPosition < str.Length;
currentStringPosition++)
try
{
char currentStringCharacter = _characterNormalizer.Normalize(str[currentStringPosition]);
patternPositionsForCurrentStringPosition.StringPosition = currentStringPosition;
patternPositionsForNextStringPosition.StringPosition = currentStringPosition + 1;
int patternPosition;
while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition))
for (int currentStringPosition = 0;
currentStringPosition < str.Length;
currentStringPosition++)
{
_patternElements[patternPosition].ProcessStringCharacter(
currentStringCharacter,
patternPosition,
patternPositionsForCurrentStringPosition,
patternPositionsForNextStringPosition);
char currentStringCharacter = _characterNormalizer.Normalize(str[currentStringPosition]);
patternPositionsForCurrentStringPosition.StringPosition = currentStringPosition;
patternPositionsForNextStringPosition.StringPosition = currentStringPosition + 1;
int patternPosition;
while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition))
{
_patternElements[patternPosition].ProcessStringCharacter(
currentStringCharacter,
patternPosition,
patternPositionsForCurrentStringPosition,
patternPositionsForNextStringPosition);
}
// swap patternPositionsForCurrentStringPosition
// with patternPositionsForNextStringPosition
var tmp = patternPositionsForCurrentStringPosition;
patternPositionsForCurrentStringPosition = patternPositionsForNextStringPosition;
patternPositionsForNextStringPosition = tmp;
}
// swap patternPositionsForCurrentStringPosition
// with patternPositionsForNextStringPosition
var tmp = patternPositionsForCurrentStringPosition;
patternPositionsForCurrentStringPosition = patternPositionsForNextStringPosition;
patternPositionsForNextStringPosition = tmp;
}
int patternPosition2;
while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition2))
{
_patternElements[patternPosition2].ProcessEndOfString(
patternPosition2,
patternPositionsForCurrentStringPosition);
}
int patternPosition2;
while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition2))
return patternPositionsForCurrentStringPosition.ReachedEndOfPattern;
}
finally
{
_patternElements[patternPosition2].ProcessEndOfString(
patternPosition2,
patternPositionsForCurrentStringPosition);
patternPositionsForCurrentStringPosition.Dispose();
patternPositionsForNextStringPosition.Dispose();
}
return patternPositionsForCurrentStringPosition.ReachedEndOfPattern;
}
private class PatternPositionsVisitor
private class PatternPositionsVisitor : IDisposable
{
private readonly int _lengthOfPattern;
@@ -965,16 +974,22 @@ namespace System.Management.Automation
_lengthOfPattern = lengthOfPattern;
_isPatternPositionVisitedMarker = new int[lengthOfPattern + 1];
for (int i = 0; i < _isPatternPositionVisitedMarker.Length; i++)
_isPatternPositionVisitedMarker = ArrayPool<int>.Shared.Rent(_lengthOfPattern + 1);
for (int i = 0; i <= _lengthOfPattern; i++)
{
_isPatternPositionVisitedMarker[i] = -1;
}
_patternPositionsForFurtherProcessing = new int[lengthOfPattern];
_patternPositionsForFurtherProcessing = ArrayPool<int>.Shared.Rent(_lengthOfPattern);
_patternPositionsForFurtherProcessingCount = 0;
}
public void Dispose()
{
ArrayPool<int>.Shared.Return(_isPatternPositionVisitedMarker, clearArray: true);
ArrayPool<int>.Shared.Return(_patternPositionsForFurtherProcessing, clearArray: true);
}
public int StringPosition { private get; set; }
public void Add(int patternPosition)
@@ -984,7 +999,7 @@ namespace System.Management.Automation
patternPosition <= _lengthOfPattern,
"Caller should verify patternPosition <= this._lengthOfPattern");
// is patternPosition already visited?);
// is patternPosition already visited?
if (_isPatternPositionVisitedMarker[patternPosition] == this.StringPosition)
{
return;