diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 33a1ca5fb4..328417a8ea 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1661,6 +1661,9 @@ namespace System.Management.Automation.Language case '\n': UngetChar(); + // Detect a line comment that disguises itself to look like the beginning of a signature block. + // This could be used to hide code at the bottom of a script, since people might assume there is nothing else after the signature. + // // The token similarity threshold was chosen by instrumenting the tokenizer and // analyzing every comment from PoshCode, Technet Script Center, and Windows. // @@ -1677,15 +1680,16 @@ namespace System.Management.Automation.Language // // There were only 279 (out of 269,387) comments with a similarity of 11,12,13,14, or 15. // At a similarity of 16-77, there were thousands of comments per similarity bucket. - // - // System.IO.File.AppendAllText(@"c:\temp\signature_similarity.txt", "" + sawBeginTokenSimilarity + ":" + commentLineComparison); const string beginSignatureTextNoSpace = "sig#beginsignatureblock\n"; const int beginTokenSimilarityThreshold = 10; - // Quick exit - the comment line is more than 'threshold' longer. Therefore, + const int beginTokenSimilarityUpperBound = 34; // beginSignatureTextNoSpace.Length + beginTokenSimilarityThreshold + const int beginTokenSimilarityLowerBound = 14; // beginSignatureTextNoSpace.Length - beginTokenSimilarityThreshold + + // Quick exit - the comment line is more than 'threshold' longer, or is less than 'threshold' shorter. Therefore, // its similarity will be over the threshold. - if (commentLine.Length > (beginSignatureTextNoSpace.Length + beginTokenSimilarityThreshold)) + if (commentLine.Length > beginTokenSimilarityUpperBound || commentLine.Length < beginTokenSimilarityLowerBound) { sawBeginSig = false; } @@ -1697,10 +1701,20 @@ namespace System.Management.Automation.Language // // The average script is 14% comments and parses in about 5.05 ms with this algorithm, // about 4.45 ms with the more simplistic algorithm. - // - string commentLineComparison = commentLine.ToString().ToLowerInvariant(); - int sawBeginTokenSimilarity = GetStringSimilarity(commentLineComparison, beginSignatureTextNoSpace); + string commentLineComparison = commentLine.ToString().ToLowerInvariant(); + if (_beginTokenSimilarity2dArray == null) + { + // Create the 2 dimensional array for edit distance calculation if it hasn't been created yet. + _beginTokenSimilarity2dArray = new int[beginTokenSimilarityUpperBound + 1, beginSignatureTextNoSpace.Length + 1]; + } + else + { + // Zero out the 2 dimensional array before using it. + Array.Clear(_beginTokenSimilarity2dArray, 0, _beginTokenSimilarity2dArray.Length); + } + + int sawBeginTokenSimilarity = GetStringSimilarity(commentLineComparison, beginSignatureTextNoSpace, _beginTokenSimilarity2dArray); sawBeginSig = sawBeginTokenSimilarity < beginTokenSimilarityThreshold; } @@ -1717,6 +1731,9 @@ namespace System.Management.Automation.Language #region Object reuse + // A two-dimensional integer array reused for calculating string similarity. + private int[,] _beginTokenSimilarity2dArray; + private readonly Queue _stringBuilders = new Queue(); private StringBuilder GetStringBuilder() @@ -1796,14 +1813,14 @@ namespace System.Management.Automation.Language // Implementation of the Levenshtein Distance algorithm // https://en.wikipedia.org/wiki/Levenshtein_distance - private static int GetStringSimilarity(string first, string second) + private static int GetStringSimilarity(string first, string second, int[,] distanceMap = null) { Diagnostics.Assert(!string.IsNullOrEmpty(first) && !string.IsNullOrEmpty(second), "Caller never calls us with empty strings"); // Store a distance map to store the number of edits required to // convert the first letters of First to the first // letters of Second. - int[,] distanceMap = new int[first.Length + 1, second.Length + 1]; + distanceMap ??= new int[first.Length + 1, second.Length + 1]; // Initialize the first row and column of the matrix - the number // of edits required when one of the strings is empty is just diff --git a/src/System.Management.Automation/engine/regex.cs b/src/System.Management.Automation/engine/regex.cs index 238cf2cbc1..cd0f1aa12e 100644 --- a/src/System.Management.Automation/engine/regex.cs +++ b/src/System.Management.Automation/engine/regex.cs @@ -3,6 +3,7 @@ #pragma warning disable 1634, 1691 +using System.Buffers; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Globalization; @@ -914,43 +915,51 @@ namespace System.Management.Automation var patternPositionsForNextStringPosition = new PatternPositionsVisitor(_patternElements.Length); - for (int currentStringPosition = 0; - currentStringPosition < str.Length; - currentStringPosition++) + try { - char currentStringCharacter = _characterNormalizer.Normalize(str[currentStringPosition]); - patternPositionsForCurrentStringPosition.StringPosition = currentStringPosition; - patternPositionsForNextStringPosition.StringPosition = currentStringPosition + 1; - - int patternPosition; - while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition)) + for (int currentStringPosition = 0; + currentStringPosition < str.Length; + currentStringPosition++) { - _patternElements[patternPosition].ProcessStringCharacter( - currentStringCharacter, - patternPosition, - patternPositionsForCurrentStringPosition, - patternPositionsForNextStringPosition); + char currentStringCharacter = _characterNormalizer.Normalize(str[currentStringPosition]); + patternPositionsForCurrentStringPosition.StringPosition = currentStringPosition; + patternPositionsForNextStringPosition.StringPosition = currentStringPosition + 1; + + int patternPosition; + while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition)) + { + _patternElements[patternPosition].ProcessStringCharacter( + currentStringCharacter, + patternPosition, + patternPositionsForCurrentStringPosition, + patternPositionsForNextStringPosition); + } + + // swap patternPositionsForCurrentStringPosition + // with patternPositionsForNextStringPosition + var tmp = patternPositionsForCurrentStringPosition; + patternPositionsForCurrentStringPosition = patternPositionsForNextStringPosition; + patternPositionsForNextStringPosition = tmp; } - // swap patternPositionsForCurrentStringPosition - // with patternPositionsForNextStringPosition - var tmp = patternPositionsForCurrentStringPosition; - patternPositionsForCurrentStringPosition = patternPositionsForNextStringPosition; - patternPositionsForNextStringPosition = tmp; - } + int patternPosition2; + while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition2)) + { + _patternElements[patternPosition2].ProcessEndOfString( + patternPosition2, + patternPositionsForCurrentStringPosition); + } - int patternPosition2; - while (patternPositionsForCurrentStringPosition.MoveNext(out patternPosition2)) + return patternPositionsForCurrentStringPosition.ReachedEndOfPattern; + } + finally { - _patternElements[patternPosition2].ProcessEndOfString( - patternPosition2, - patternPositionsForCurrentStringPosition); + patternPositionsForCurrentStringPosition.Dispose(); + patternPositionsForNextStringPosition.Dispose(); } - - return patternPositionsForCurrentStringPosition.ReachedEndOfPattern; } - private class PatternPositionsVisitor + private class PatternPositionsVisitor : IDisposable { private readonly int _lengthOfPattern; @@ -965,16 +974,22 @@ namespace System.Management.Automation _lengthOfPattern = lengthOfPattern; - _isPatternPositionVisitedMarker = new int[lengthOfPattern + 1]; - for (int i = 0; i < _isPatternPositionVisitedMarker.Length; i++) + _isPatternPositionVisitedMarker = ArrayPool.Shared.Rent(_lengthOfPattern + 1); + for (int i = 0; i <= _lengthOfPattern; i++) { _isPatternPositionVisitedMarker[i] = -1; } - _patternPositionsForFurtherProcessing = new int[lengthOfPattern]; + _patternPositionsForFurtherProcessing = ArrayPool.Shared.Rent(_lengthOfPattern); _patternPositionsForFurtherProcessingCount = 0; } + public void Dispose() + { + ArrayPool.Shared.Return(_isPatternPositionVisitedMarker, clearArray: true); + ArrayPool.Shared.Return(_patternPositionsForFurtherProcessing, clearArray: true); + } + public int StringPosition { private get; set; } public void Add(int patternPosition) @@ -984,7 +999,7 @@ namespace System.Management.Automation patternPosition <= _lengthOfPattern, "Caller should verify patternPosition <= this._lengthOfPattern"); - // is patternPosition already visited?); + // is patternPosition already visited? if (_isPatternPositionVisitedMarker[patternPosition] == this.StringPosition) { return;