Make Get-ChildItem follow symlinks on demand, with checks for link loops (#4020)

Add the dynamic parameter `-FollowSymlink` to `Get-ChildItem`.
Add a mechanism for tracking visited directories.
Add native code to get device/inode information on Unix/Windows.
Add warning when refusing to enter an already-visited directory.

src/System.Management.Automation/CoreCLR/CorePsPlatform.cs

@@ -587,6 +587,16 @@ namespace System.Management.Automation
             return Unix.NativeMethods.IsSameFileSystemItem(pathOne, pathTwo);
         }
 
+        internal static bool NonWindowsGetInodeData(string path, out System.ValueTuple<UInt64, UInt64> inodeData)
+        {
+            UInt64 device = 0UL;
+            UInt64 inode = 0UL;
+            var result = Unix.NativeMethods.GetInodeData(path, out device, out inode);
+
+            inodeData = (device, inode);
+            return result == 0;
+        }
+
         internal static bool NonWindowsIsExecutable(string path)
         {
             return Unix.NativeMethods.IsExecutable(path);
@@ -797,6 +807,10 @@ namespace System.Management.Automation
                 [return: MarshalAs(UnmanagedType.I1)]
                 internal static extern bool IsSameFileSystemItem([MarshalAs(UnmanagedType.LPStr)]string filePathOne,
                                                                  [MarshalAs(UnmanagedType.LPStr)]string filePathTwo);
+
+                [DllImport(psLib, CharSet = CharSet.Ansi, SetLastError = true)]
+                internal static extern int GetInodeData([MarshalAs(UnmanagedType.LPStr)]string path,
+                                                        out UInt64 device, out UInt64 inode);
             }
         }
     }

src/System.Management.Automation/namespaces/FileSystemProvider.cs

@@ -1539,8 +1539,19 @@ namespace Microsoft.PowerShell.Commands
                 if (isDirectory)
                 {
                     DirectoryInfo directory = new DirectoryInfo(path);
+                    InodeTracker tracker = null;
+
+                    if (recurse)
+                    {
+                        GetChildDynamicParameters fspDynamicParam = DynamicParameters as GetChildDynamicParameters;
+                        if (fspDynamicParam != null && fspDynamicParam.FollowSymlink)
+                        {
+                            tracker = new InodeTracker(directory.FullName);
+                        }
+                    }
+
                     // Enumerate the directory
-                    Dir(directory, recurse, depth, nameOnly, returnContainers);
+                    Dir(directory, recurse, depth, nameOnly, returnContainers, tracker);
                 }
                 else
                 {
@@ -1609,7 +1620,8 @@ namespace Microsoft.PowerShell.Commands
             bool recurse,
             uint depth,
             bool nameOnly,
-            ReturnContainers returnContainers)
+            ReturnContainers returnContainers,
+            InodeTracker tracker)   // tracker will be non-null only if the user invoked the -FollowSymLinks and -Recurse switch parameters.
         {
             List<IEnumerable<FileSystemInfo>> target = new List<IEnumerable<FileSystemInfo>>();
 
@@ -1788,25 +1800,35 @@ namespace Microsoft.PowerShell.Commands
                                 return;
                             }
 
-                            // Once the recursion process has begun by being in this function,
-                            // we do not want to further recurse into directory symbolic links
-                            // so as to prevent the possibility of an endless symlink loop.
-                            // This is the behavior of both the Unix 'ls' command and the Windows
-                            // 'DIR' command.
-                            if (!InternalSymbolicLinkLinkCodeMethods.IsReparsePoint(recursiveDirectory))
+                            bool hidden = false;
+                            if (!Force)
                             {
-                                bool hidden = false;
-                                if (!Force)
+                                hidden = (recursiveDirectory.Attributes & FileAttributes.Hidden) != 0;
+                            }
+
+                            // if "Hidden" is explicitly specified anywhere in the attribute filter, then override
+                            // default hidden attribute filter.
+                            if (Force || !hidden || isFilterHiddenSpecified || isSwitchFilterHiddenSpecified)
+                            {
+                                // We only want to recurse into symlinks if
+                                //  a) the user has asked to with the -FollowSymLinks switch parameter and
+                                //  b) the directory pointed to by the symlink has not already been visited,
+                                //     preventing symlink loops.
+                                if (tracker == null)
                                 {
-                                    hidden = (recursiveDirectory.Attributes & FileAttributes.Hidden) != 0;
+                                    if (InternalSymbolicLinkLinkCodeMethods.IsReparsePoint(recursiveDirectory))
+                                    {
+                                        continue;
+                                    }
+                                }
+                                else if (!tracker.TryVisitPath(recursiveDirectory.FullName))
+                                {
+                                    WriteWarning(StringUtil.Format(FileSystemProviderStrings.AlreadyListedDirectory,
+                                                                   recursiveDirectory.FullName));
+                                    continue;
                                 }
 
-                                // if "Hidden" is explicitly specified anywhere in the attribute filter, then override
-                                // default hidden attribute filter.
-                                if (Force || !hidden || isFilterHiddenSpecified || isSwitchFilterHiddenSpecified)
-                                {
-                                    Dir(recursiveDirectory, recurse, depth - 1, nameOnly, returnContainers);
-                                }
+                                Dir(recursiveDirectory, recurse, depth - 1, nameOnly, returnContainers, tracker);
                             }
                         }//foreach
                     }//if
@@ -7320,6 +7342,52 @@ namespace Microsoft.PowerShell.Commands
             [MarshalAs(UnmanagedType.LPWStr)]
             public string Provider;
         }
+
+        #region InodeTracker
+        /// <summary>
+        /// Tracks visited files/directories by caching their device IDs and inodes.
+        /// </summary>
+        private class InodeTracker
+        {
+            private HashSet<(UInt64, UInt64)> _visitations;
+
+            /// <summary>
+            /// Construct a new InodeTracker with an initial path
+            /// </summary>
+            internal InodeTracker(string path)
+            {
+                _visitations = new HashSet<(UInt64, UInt64)>();
+
+                if (InternalSymbolicLinkLinkCodeMethods.GetInodeData(path, out (UInt64, UInt64) inodeData))
+                {
+                    _visitations.Add(inodeData);
+                }
+            }
+
+            /// <summary>
+            /// Attempt to mark a path as having been visited.
+            /// </summary>
+            /// <param name="path">
+            /// Path to the file system item to be visited.
+            /// </param>
+            /// <returns>
+            /// True if the path had not been previously visited and was
+            /// successfully marked as visited, false otherwise.
+            /// </returns>
+            internal bool TryVisitPath(string path)
+            {
+                bool returnValue = false;
+
+                if (InternalSymbolicLinkLinkCodeMethods.GetInodeData(path, out (UInt64, UInt64) inodeData))
+                {
+                    returnValue = _visitations.Add(inodeData);
+                }
+
+                return returnValue;
+            }
+        }
+
+        #endregion
     } // class FileSystemProvider
 
     internal static class SafeInvokeCommand
@@ -7482,6 +7550,12 @@ namespace Microsoft.PowerShell.Commands
         [Parameter]
         public FlagsExpression<FileAttributes> Attributes { get; set; }
 
+        /// <summary>
+        /// Gets or sets the flag to follow symbolic links when recursing.
+        /// </summary>
+        [Parameter]
+        public SwitchParameter FollowSymlink { get; set; }
+
         /// <summary>
         /// Gets or sets the filter directory flag
         /// </summary>
@@ -8238,6 +8312,45 @@ namespace Microsoft.PowerShell.Commands
             return false;
         }
 
+        internal static bool GetInodeData(string path, out System.ValueTuple<UInt64, UInt64> inodeData)
+        {
+#if UNIX
+            bool rv = Platform.NonWindowsGetInodeData(path, out inodeData);
+#else
+            bool rv = WinGetInodeData(path, out inodeData);
+#endif
+            return rv;
+        }
+
+        internal static bool WinGetInodeData(string path, out System.ValueTuple<UInt64, UInt64> inodeData)
+        {
+            var access = FileAccess.Read;
+            var share = FileShare.Read;
+            var creation = FileMode.Open;
+            var attributes = FileAttributes.BackupSemantics | FileAttributes.PosixSemantics;
+
+            using (var sf = AlternateDataStreamUtilities.NativeMethods.CreateFile(path, access, share, IntPtr.Zero, creation, (int)attributes, IntPtr.Zero))
+            {
+                if (!sf.IsInvalid)
+                {
+                    BY_HANDLE_FILE_INFORMATION info;
+
+                    if (GetFileInformationByHandle(sf.DangerousGetHandle(), out info))
+                    {
+                        UInt64 tmp = info.FileIndexHigh;
+                        tmp = (tmp << 32) | info.FileIndexLow;
+
+                        inodeData = (info.VolumeSerialNumber, tmp);
+
+                        return true;
+                    }
+                }
+            }
+
+            inodeData = (0, 0);
+            return false;
+        }
+
         internal static bool IsHardLink(ref IntPtr handle)
         {
 #if UNIX

src/System.Management.Automation/resources/FileSystemProviderStrings.resx

@@ -339,4 +339,7 @@
   <data name="SymlinkItemExists" xml:space="preserve">
     <value>Cannot create symbolic link because the path {0} already exists.</value>
   </data>
+  <data name="AlreadyListedDirectory" xml:space="preserve">
+    <value>Skip already-visited directory {0}.</value>
+  </data>
 </root>

src/libpsl-native/src/CMakeLists.txt

@@ -11,6 +11,7 @@ add_library(psl-native SHARED
   getlinkcount.cpp
   getfullyqualifiedname.cpp
   geterrorcategory.cpp
+  getinodedata.cpp
   isfile.cpp
   isdirectory.cpp
   issamefilesystemitem.cpp

src/libpsl-native/src/getinodedata.cpp

@@ -0,0 +1,56 @@
+//! @file getinodedata.cpp
+//! @author Jeff Bienstadt <v-jebien@microsoft.com>
+//! @brief Retrieve the device ID and inode number of a file
+
+#include "getinodedata.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <locale.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <string>
+
+//! @brief GetInodeData retrieves a file's device and inode information.
+//!
+//! GetInodeData
+//!
+//! @param[in] fileName
+//! @parblock
+//! A pointer to the buffer that contains the file path
+//!
+//! char* is marshaled as an LPStr, which on Linux is UTF-8.
+//! @endparblock
+//!
+//! @param[out] device
+//! @parblock
+//! Points to a uint64_t value that will contain the file's device ID.
+//! @endparblock
+//!
+//! @param[out] inode
+//! @parblock
+//! Points to a uint64_t value that will contain the file's inode number.
+//! @endparblock
+//!
+//! @retval 0 If the function succeeds, -1 otherwise.
+//!
+
+int32_t GetInodeData(const char* fileName, uint64_t* device, uint64_t* inode)
+{
+    assert(fileName);
+    assert(device);
+    assert(inode);
+    errno = 0;
+
+    struct stat statBuf;
+    int ret = stat(fileName, &statBuf);
+
+    if (ret == 0)
+    {
+        *device = statBuf.st_dev;
+        *inode = statBuf.st_ino;
+    }
+
+    return ret;
+}

src/libpsl-native/src/getinodedata.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "pal.h"
+
+PAL_BEGIN_EXTERNC
+
+int32_t GetInodeData(const char* fileName, uint64_t* device, uint64_t* inode);
+
+PAL_END_EXTERNC

test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1

@@ -376,11 +376,18 @@ Describe "Hard link and symbolic link tests" -Tags "CI", "RequireAdminOnWindows"
             $alphaLink = Join-Path $TestDrive "link-alpha"
             $alphaFile1 = Join-Path $alphaDir "AlphaFile1.txt"
             $alphaFile2 = Join-Path $alphaDir "AlphaFile2.txt"
+            $omegaDir = Join-Path $TestDrive "sub-omega"
+            $omegaFile1 = Join-Path $omegaDir "OmegaFile1"
+            $omegaFile2 = Join-Path $omegaDir "OmegaFile2"
             $betaDir = Join-Path $alphaDir "sub-beta"
             $betaLink = Join-Path $alphaDir "link-beta"
             $betaFile1 = Join-Path $betaDir "BetaFile1.txt"
             $betaFile2 = Join-Path $betaDir "BetaFile2.txt"
             $betaFile3 = Join-Path $betaDir "BetaFile3.txt"
+            $gammaDir = Join-Path $betaDir "sub-gamma"
+            $uponeLink = Join-Path $gammaDir "upone-link"
+            $uptwoLink = Join-Path $gammaDir "uptwo-link"
+            $omegaLink = Join-Path $gammaDir "omegaLink"
 
             New-Item -ItemType Directory -Path $alphaDir
             New-Item -ItemType File -Path $alphaFile1
@@ -389,6 +396,9 @@ Describe "Hard link and symbolic link tests" -Tags "CI", "RequireAdminOnWindows"
             New-Item -ItemType File -Path $betaFile1
             New-Item -ItemType File -Path $betaFile2
             New-Item -ItemType File -Path $betaFile3
+            New-Item -ItemType Directory $omegaDir
+            New-Item -ItemType File -Path $omegaFile1
+            New-Item -ItemType File -Path $omegaFile2
         }
         AfterAll {
             Remove-Item -Path $alphaLink -Force -ErrorAction SilentlyContinue
@@ -408,6 +418,15 @@ Describe "Hard link and symbolic link tests" -Tags "CI", "RequireAdminOnWindows"
             $ci = Get-ChildItem $alphaLink -Recurse
             $ci.Count | Should BeExactly 7
         }
+        It "Get-ChildItem will recurse into symlinks given -FollowSymlink, avoiding link loops" {
+            New-Item -ItemType Directory -Path $gammaDir
+            New-Item -ItemType SymbolicLink -Path $uponeLink -Value $betaDir
+            New-Item -ItemType SymbolicLink -Path $uptwoLink -Value $alphaDir
+            New-Item -ItemType SymbolicLink -Path $omegaLink -Value $omegaDir
+            $ci = Get-ChildItem -Path $alphaDir -FollowSymlink -Recurse -WarningVariable w -WarningAction SilentlyContinue
+            $ci.Count | Should BeExactly 13
+            $w.Count | Should BeExactly 3
+        }
     }
 
     Context "Remove-Item and hard/symbolic links" {