Support byte stream piping between native commands and file redirection (#17857)

src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#nullable enable
+
+using System.Buffers;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace System.Management.Automation;
+
+/// <summary>
+/// Represents the transfer of bytes from one <see cref="Stream" /> to another
+/// asynchronously.
+/// </summary>
+internal sealed class AsyncByteStreamTransfer : IDisposable
+{
+    private const int DefaultBufferSize = 1024;
+
+    private readonly BytePipe _bytePipe;
+
+    private readonly BytePipe _destinationPipe;
+
+    private readonly Memory<byte> _buffer;
+
+    private readonly CancellationTokenSource _cts = new();
+
+    private Task? _readToBufferTask;
+
+    public AsyncByteStreamTransfer(
+        BytePipe bytePipe,
+        BytePipe destinationPipe)
+    {
+        _bytePipe = bytePipe;
+        _destinationPipe = destinationPipe;
+        _buffer = new byte[DefaultBufferSize];
+    }
+
+    public Task EOF => _readToBufferTask ?? Task.CompletedTask;
+
+    public void BeginReadChunks()
+    {
+        _readToBufferTask = Task.Run(ReadBufferAsync);
+    }
+
+    public void Dispose() => _cts.Cancel();
+
+    private async Task ReadBufferAsync()
+    {
+        Stream stream;
+        Stream? destinationStream = null;
+        try
+        {
+            stream = await _bytePipe.GetStream(_cts.Token);
+            destinationStream = await _destinationPipe.GetStream(_cts.Token);
+
+            while (true)
+            {
+                int bytesRead;
+                bytesRead = await stream.ReadAsync(_buffer, _cts.Token);
+                if (bytesRead is 0)
+                {
+                    break;
+                }
+
+                destinationStream.Write(_buffer.Span.Slice(0, bytesRead));
+            }
+        }
+        catch (IOException)
+        {
+            return;
+        }
+        catch (OperationCanceledException)
+        {
+            return;
+        }
+        finally
+        {
+            destinationStream?.Close();
+        }
+    }
+}

src/System.Management.Automation/engine/BytePipe.cs

@@ -0,0 +1,115 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#nullable enable
+
+using System.Diagnostics;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerShell.Telemetry;
+
+namespace System.Management.Automation;
+
+/// <summary>
+/// Represents a lazily retrieved <see cref="Stream" /> for transfering bytes
+/// to or from.
+/// </summary>
+internal abstract class BytePipe
+{
+    public abstract Task<Stream> GetStream(CancellationToken cancellationToken);
+
+    internal AsyncByteStreamTransfer Bind(BytePipe bytePipe)
+    {
+        Debug.Assert(bytePipe is not null);
+        return new AsyncByteStreamTransfer(bytePipe, destinationPipe: this);
+    }
+}
+
+/// <summary>
+/// Represents a <see cref="Stream" /> lazily retrieved from the underlying
+/// <see cref="NativeCommandProcessor" />.
+/// </summary>
+internal sealed class NativeCommandProcessorBytePipe : BytePipe
+{
+    private readonly NativeCommandProcessor _nativeCommand;
+
+    private readonly bool _stdout;
+
+    internal NativeCommandProcessorBytePipe(
+        NativeCommandProcessor nativeCommand,
+        bool stdout)
+    {
+        Debug.Assert(nativeCommand is not null);
+        _nativeCommand = nativeCommand;
+        _stdout = stdout;
+    }
+
+    public override async Task<Stream> GetStream(CancellationToken cancellationToken)
+    {
+        // If the native command we're wrapping is the upstream command then
+        // NativeCommandProcessor.Prepare will have already been called before
+        // the creation of this BytePipe.
+        if (_stdout)
+        {
+            return _nativeCommand.GetStream(stdout: true);
+        }
+
+        await _nativeCommand.WaitForProcessInitializationAsync(cancellationToken);
+        return _nativeCommand.GetStream(stdout: false);
+    }
+}
+
+/// <summary>
+/// Provides an byte pipe implementation representing a <see cref="FileStream" />.
+/// </summary>
+internal sealed class FileBytePipe : BytePipe
+{
+    private readonly Stream _stream;
+
+    private FileBytePipe(Stream stream)
+    {
+        Debug.Assert(stream is not null);
+        _stream = stream;
+    }
+
+    internal static FileBytePipe Create(string fileName, bool append)
+    {
+        FileStream fileStream;
+        try
+        {
+            PathUtils.MasterStreamOpen(
+                fileName,
+                resolvedEncoding: null,
+                defaultEncoding: false,
+                append,
+                Force: true,
+                NoClobber: false,
+                out fileStream,
+                streamWriter: out _,
+                readOnlyFileInfo: out _,
+                isLiteralPath: true);
+        }
+        catch (Exception e) when (e.Data.Contains(typeof(ErrorRecord)))
+        {
+            // The error record is attached to the exception when thrown to preserve
+            // the call stack.
+            ErrorRecord? errorRecord = e.Data[typeof(ErrorRecord)] as ErrorRecord;
+            if (errorRecord is null)
+            {
+                throw;
+            }
+
+            e.Data.Remove(typeof(ErrorRecord));
+            throw new RuntimeException(null, e, errorRecord);
+        }
+
+        ApplicationInsightsTelemetry.SendExperimentalUseData(
+            ExperimentalFeature.PSNativeCommandPreserveBytePipe,
+            "f");
+
+        return new FileBytePipe(fileStream);
+    }
+
+    public override Task<Stream> GetStream(CancellationToken cancellationToken) => Task.FromResult(_stream);
+}

src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs

@@ -22,6 +22,7 @@ namespace System.Management.Automation
 
         internal const string EngineSource = "PSEngine";
         internal const string PSNativeCommandErrorActionPreferenceFeatureName = "PSNativeCommandErrorActionPreference";
+        internal const string PSNativeCommandPreserveBytePipe = "PSNativeCommandPreserveBytePipe";
         internal const string PSModuleAutoLoadSkipOfflineFilesFeatureName = "PSModuleAutoLoadSkipOfflineFiles";
         internal const string PSCustomTableHeaderLabelDecoration = "PSCustomTableHeaderLabelDecoration";
         internal const string PSFeedbackProvider = "PSFeedbackProvider";
@@ -126,6 +127,9 @@ namespace System.Management.Automation
                 new ExperimentalFeature(
                     name: PSCustomTableHeaderLabelDecoration,
                     description: "Formatting differentiation for table header labels that aren't property members"),
+                new ExperimentalFeature(
+                    name: PSNativeCommandPreserveBytePipe,
+                    description: "Byte output is retained when piping between two or more native commands"),
                 new ExperimentalFeature(
                     name: PSFeedbackProvider,
                     description: "Replace the hard-coded suggestion framework with the extensible feedback provider"),

src/System.Management.Automation/engine/NativeCommandProcessor.cs

@@ -17,6 +17,7 @@ using System.Runtime.InteropServices;
 using System.Runtime.Serialization;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using System.Xml;
 using Microsoft.PowerShell.Telemetry;
 using Dbg = System.Management.Automation.Diagnostics;
@@ -339,16 +340,16 @@ namespace System.Management.Automation
             }
         }
 
+        internal NativeCommandProcessor DownStreamNativeCommand { get; set; }
+
+        internal bool UpstreamIsNativeCommand { get; set; }
+
+        internal BytePipe StdOutDestination { get; set; }
+
         #endregion ctor/native command properties
 
         #region parameter binder
 
-        /// <summary>
-        /// Variable which is set to true when prepare is called.
-        /// Parameter Binder should only be created after Prepare method is called.
-        /// </summary>
-        private bool _isPreparedCalled = false;
-
         /// <summary>
         /// Parameter binder used by this command processor.
         /// </summary>
@@ -365,8 +366,6 @@ namespace System.Management.Automation
         /// </returns>
         internal ParameterBinderController NewParameterBinderController(InternalCommand command)
         {
-            Dbg.Assert(_isPreparedCalled, "parameter binder should not be created before prepared is called");
-
             if (_isMiniShell)
             {
                 _nativeParameterBinderController =
@@ -405,8 +404,6 @@ namespace System.Management.Automation
         /// </summary>
         internal override void Prepare(IDictionary psDefaultParameterValues)
         {
-            _isPreparedCalled = true;
-
             // Check if the application is minishell
             _isMiniShell = IsMiniShell();
 
@@ -436,9 +433,15 @@ namespace System.Management.Automation
         {
             try
             {
-                while (Read())
+                // If upstream is a native command it'll be writing directly to our stdin stream
+                // so we can skip reading here.
+                if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                    && !UpstreamIsNativeCommand)
                 {
-                    _inputWriter.Add(Command.CurrentPipelineObject);
+                    while (Read())
+                    {
+                        _inputWriter.Add(Command.CurrentPipelineObject);
+                    }
                 }
 
                 ConsumeAvailableNativeProcessOutput(blocking: false);
@@ -496,6 +499,59 @@ namespace System.Management.Automation
         /// </summary>
         private readonly object _sync = new object();
 
+        private SemaphoreSlim _processInitialized;
+
+        internal async Task WaitForProcessInitializationAsync(CancellationToken cancellationToken)
+        {
+            SemaphoreSlim processInitialized = _processInitialized;
+            if (processInitialized is null)
+            {
+                lock (_sync)
+                {
+                    processInitialized = _processInitialized ??= new SemaphoreSlim(0, 1);
+                }
+            }
+
+            try
+            {
+                await processInitialized.WaitAsync(cancellationToken);
+            }
+            finally
+            {
+                processInitialized.Release();
+            }
+        }
+
+        /// <summary>
+        /// Creates a pipe representing the streaming of unprocessed bytes.
+        /// </summary>
+        /// <param name="stdout">
+        /// The stream that the pipe should represent. <see langword="true" />
+        /// for stdout, <see langword="false" /> for stdin.
+        /// </param>
+        /// <returns>A new byte pipe representing the specified stream.</returns>
+        internal BytePipe CreateBytePipe(bool stdout) => new NativeCommandProcessorBytePipe(this, stdout);
+
+        /// <summary>
+        /// Gets the specified base <see cref="Stream" /> for the underlying
+        /// <see cref="Process" />.
+        /// </summary>
+        /// <param name="stdout">
+        /// The stream that should be retrieved. <see langword="true" /> for
+        /// stdout, <see langword="false" /> for stdin.
+        /// </param>
+        /// <returns>The specified <see cref="Stream" />.</returns>
+        internal Stream GetStream(bool stdout)
+        {
+            Debug.Assert(
+                _nativeProcess is not null,
+                "Caller should verify that initialization has completed before attempting to get the underlying stream.");
+
+            return stdout
+                ? _nativeProcess.StandardOutput.BaseStream
+                : _nativeProcess.StandardInput.BaseStream;
+        }
+
         /// <summary>
         /// Executes the native command once all of the input has been gathered.
         /// </summary>
@@ -581,6 +637,19 @@ namespace System.Management.Automation
                     {
                         _nativeProcess = new Process() { StartInfo = startInfo };
                         _nativeProcess.Start();
+                        if (UpstreamIsNativeCommand)
+                        {
+                            SemaphoreSlim processInitialized = _processInitialized;
+                            if (processInitialized is null)
+                            {
+                                lock (_sync)
+                                {
+                                    processInitialized = _processInitialized ??= new SemaphoreSlim(0, 1);
+                                }
+                            }
+
+                            processInitialized?.Release();
+                        }
                     }
                     catch (Win32Exception)
                     {
@@ -691,7 +760,9 @@ namespace System.Management.Automation
 
                         lock (_sync)
                         {
-                            if (!_stopped)
+                            if (!_stopped
+                                && (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                                || !UpstreamIsNativeCommand))
                             {
                                 _inputWriter.Start(_nativeProcess, inputFormat);
                             }
@@ -740,6 +811,8 @@ namespace System.Management.Automation
             }
         }
 
+        private AsyncByteStreamTransfer _stdOutByteTransfer;
+
         private void InitOutputQueue()
         {
             // if output is redirected, start reading output of process in queue.
@@ -749,9 +822,38 @@ namespace System.Management.Automation
                 {
                     if (!_stopped)
                     {
+                        if (CommandRuntime.ErrorMergeTo is MshCommandRuntime.MergeDataStream.Output)
+                        {
+                            StdOutDestination = null;
+                            if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe))
+                            {
+                                if (DownStreamNativeCommand is not null)
+                                {
+                                    DownStreamNativeCommand.UpstreamIsNativeCommand = false;
+                                    DownStreamNativeCommand = null;
+                                }
+                            }
+                        }
+
                         _nativeProcessOutputQueue = new BlockingCollection<ProcessOutputObject>();
                         // we don't assign the handler to anything, because it's used only for objects marshaling
-                        new ProcessOutputHandler(_nativeProcess, _nativeProcessOutputQueue);
+                        BytePipe stdOutDestination = ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                            ? StdOutDestination ?? DownStreamNativeCommand?.CreateBytePipe(stdout: false)
+                            : null;
+
+                        BytePipe stdOutSource = null;
+                        if (stdOutDestination is not null)
+                        {
+                            stdOutSource = CreateBytePipe(stdout: true);
+                        }
+
+                        _ = new ProcessOutputHandler(
+                            _nativeProcess,
+                            _nativeProcessOutputQueue,
+                            stdOutDestination,
+                            stdOutSource,
+                            out _stdOutByteTransfer);
+
                     }
                 }
             }
@@ -761,6 +863,13 @@ namespace System.Management.Automation
         {
             if (blocking)
             {
+                if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                    && _stdOutByteTransfer is not null)
+                {
+                    _stdOutByteTransfer.EOF.GetAwaiter().GetResult();
+                    return null;
+                }
+
                 // If adding was completed and collection is empty (IsCompleted == true)
                 // there is no need to do a blocking Take(), we should just return.
                 if (!_nativeProcessOutputQueue.IsCompleted)
@@ -784,6 +893,12 @@ namespace System.Management.Automation
             }
             else
             {
+                if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                    && _stdOutByteTransfer is not null)
+                {
+                    return null;
+                }
+
                 ProcessOutputObject record = null;
                 _nativeProcessOutputQueue.TryTake(out record);
                 return record;
@@ -822,7 +937,11 @@ namespace System.Management.Automation
                 if (!_isRunningInBackground)
                 {
                     // Wait for input writer to finish.
-                    _inputWriter.Done();
+                    if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                        || !UpstreamIsNativeCommand)
+                    {
+                        _inputWriter.Done();
+                    }
 
                     // read all the available output in the blocking way
                     ConsumeAvailableNativeProcessOutput(blocking: true);
@@ -1188,8 +1307,13 @@ namespace System.Management.Automation
                 if (!_runStandAlone)
                 {
                     // Stop input writer
-                    _inputWriter.Stop();
+                    if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                        || !UpstreamIsNativeCommand)
+                    {
+                        _inputWriter.Stop();
+                    }
 
+                    _stdOutByteTransfer?.Dispose();
                     KillProcess(_nativeProcess);
                 }
             }
@@ -1675,7 +1799,19 @@ namespace System.Management.Automation
         private bool _isXmlCliError;
         private readonly string _processFileName;
 
+        private readonly AsyncByteStreamTransfer _stdOutDrainer;
+
         public ProcessOutputHandler(Process process, BlockingCollection<ProcessOutputObject> queue)
+            : this(process, queue, null, null, out _)
+        {
+        }
+
+        public ProcessOutputHandler(
+            Process process,
+            BlockingCollection<ProcessOutputObject> queue,
+            BytePipe stdOutDestination,
+            BytePipe stdOutSource,
+            out AsyncByteStreamTransfer stdOutDrainer)
         {
             Debug.Assert(process.StartInfo.RedirectStandardOutput || process.StartInfo.RedirectStandardError, "Caller should redirect at least one stream");
             _refCount = 0;
@@ -1695,14 +1831,6 @@ namespace System.Management.Automation
             }
 
             // once we have _refCount, we can start processing
-            if (process.StartInfo.RedirectStandardOutput)
-            {
-                _isFirstOutput = true;
-                _isXmlCliOutput = false;
-                process.OutputDataReceived += OutputHandler;
-                process.BeginOutputReadLine();
-            }
-
             if (process.StartInfo.RedirectStandardError)
             {
                 _isFirstError = true;
@@ -1710,6 +1838,26 @@ namespace System.Management.Automation
                 process.ErrorDataReceived += ErrorHandler;
                 process.BeginErrorReadLine();
             }
+
+            stdOutDrainer = null;
+
+            if (!process.StartInfo.RedirectStandardOutput)
+            {
+                return;
+            }
+
+            if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                || stdOutDestination is null)
+            {
+                _isFirstOutput = true;
+                _isXmlCliOutput = false;
+                process.OutputDataReceived += OutputHandler;
+                process.BeginOutputReadLine();
+                return;
+            }
+
+            stdOutDrainer = _stdOutDrainer = stdOutDestination.Bind(stdOutSource);
+            stdOutDrainer.BeginReadChunks();
         }
 
         private void decrementRefCount()
@@ -1948,14 +2096,30 @@ namespace System.Management.Automation
                 return;
             }
 
-            if (_inputFormat == NativeCommandIOFormat.Text)
-            {
-                AddTextInput(input);
-            }
-            else // Xml
+            if (_inputFormat is not NativeCommandIOFormat.Text)
             {
                 AddXmlInput(input);
+                return;
             }
+
+            object baseObjInput = PSObject.Base(input);
+
+            if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe))
+            {
+                if (baseObjInput is byte[] bytes)
+                {
+                    _streamWriter.BaseStream.Write(bytes, 0, bytes.Length);
+                    return;
+                }
+
+                if (baseObjInput is byte b)
+                {
+                    _streamWriter.BaseStream.WriteByte(b);
+                    return;
+                }
+            }
+
+            AddTextInput(input);
         }
 
         private void AddTextInput(object input)

src/System.Management.Automation/engine/pipeline.cs

@@ -43,6 +43,10 @@ namespace System.Management.Automation.Internal
         private bool _linkedSuccessOutput = false;
         private bool _linkedErrorOutput = false;
 
+        private NativeCommandProcessor _lastNativeCommand;
+
+        private bool _haveReportedNativePipeUsage;
+
 #if !CORECLR // Impersonation Not Supported On CSS
         // This is the security context when the pipeline was allocated
         internal System.Security.SecurityContext SecurityContext =
@@ -257,6 +261,33 @@ namespace System.Management.Automation.Internal
         /// <exception cref="ObjectDisposedException"></exception>
         internal int Add(CommandProcessorBase commandProcessor)
         {
+            if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe))
+            {
+                if (commandProcessor is NativeCommandProcessor nativeCommand)
+                {
+                    if (_lastNativeCommand is not null)
+                    {
+                        // Only report experimental feature usage once per pipeline.
+                        if (!_haveReportedNativePipeUsage)
+                        {
+                            ApplicationInsightsTelemetry.SendExperimentalUseData(
+                                ExperimentalFeature.PSNativeCommandPreserveBytePipe,
+                                "p");
+                            _haveReportedNativePipeUsage = true;
+                        }
+
+                        _lastNativeCommand.DownStreamNativeCommand = nativeCommand;
+                        nativeCommand.UpstreamIsNativeCommand = true;
+                    }
+
+                    _lastNativeCommand = nativeCommand;
+                }
+                else
+                {
+                    _lastNativeCommand = null;
+                }
+            }
+
             commandProcessor.CommandRuntime.PipelineProcessor = this;
             return AddCommand(commandProcessor, _commands.Count, readErrorQueue: false);
         }

src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs

@@ -162,6 +162,7 @@ namespace System.Management.Automation
                                              (cmd is ScriptCommand || cmd is PSScriptCmdlet);
 
             bool isNativeCommand = commandProcessor is NativeCommandProcessor;
+
             for (int i = commandIndex + 1; i < commandElements.Length; ++i)
             {
                 var cpi = commandElements[i];
@@ -208,9 +209,27 @@ namespace System.Management.Automation
             bool redirectedInformation = false;
             if (redirections != null)
             {
-                foreach (var redirection in redirections)
+                bool shouldProcessMergesFirst = ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)
+                    && isNativeCommand;
+
+                if (shouldProcessMergesFirst)
                 {
-                    redirection.Bind(pipe, commandProcessor, context);
+                    foreach (CommandRedirection redirection in redirections)
+                    {
+                        if (redirection is MergingRedirection)
+                        {
+                            redirection.Bind(pipe, commandProcessor, context);
+                        }
+                    }
+                }
+
+                foreach (CommandRedirection redirection in redirections)
+                {
+                    if (!shouldProcessMergesFirst || redirection is not MergingRedirection)
+                    {
+                        redirection.Bind(pipe, commandProcessor, context);
+                    }
+
                     switch (redirection.FromStream)
                     {
                         case RedirectionStream.Error:
@@ -1050,6 +1069,18 @@ namespace System.Management.Automation
         //    dir > out
         internal override void Bind(PipelineProcessor pipelineProcessor, CommandProcessorBase commandProcessor, ExecutionContext context)
         {
+            if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe))
+            {
+                if (commandProcessor is NativeCommandProcessor nativeCommand
+                    && nativeCommand.CommandRuntime.ErrorMergeTo is not MshCommandRuntime.MergeDataStream.Output
+                    && FromStream is RedirectionStream.Output
+                    && !string.IsNullOrWhiteSpace(File))
+                {
+                    nativeCommand.StdOutDestination = FileBytePipe.Create(File, Appending);
+                    return;
+                }
+            }
+
             Pipe pipe = GetRedirectionPipe(context, pipelineProcessor);
 
             switch (FromStream)
@@ -3627,7 +3658,7 @@ namespace System.Management.Automation
             }
             catch (PSSecurityException)
             {
-                // ReportContent() will throw PSSecurityException if AMSI detects malware, which 
+                // ReportContent() will throw PSSecurityException if AMSI detects malware, which
                 // must be propagated.
                 throw;
             }

src/System.Management.Automation/utils/PathUtils.cs

@@ -6,6 +6,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.IO;
 using System.Management.Automation.Internal;
+using System.Management.Automation.Runspaces;
 using System.Runtime.CompilerServices;
 using System.Text;
 using Microsoft.PowerShell.Commands;
@@ -83,55 +84,19 @@ namespace System.Management.Automation
             fileStream = null;
             streamWriter = null;
             readOnlyFileInfo = null;
-
-            // resolve the path and the encoding
             string resolvedPath = ResolveFilePath(filePath, cmdlet, isLiteralPath);
-
             try
             {
-                // variable to track file open mode
-                // this is controlled by append/force parameters
-                FileMode mode = FileMode.Create;
-                if (Append)
-                {
-                    mode = FileMode.Append;
-                }
-                else if (NoClobber)
-                {
-                    // throw IOException if file exists
-                    mode = FileMode.CreateNew;
-                }
-
-                if (Force && (Append || !NoClobber))
-                {
-                    if (File.Exists(resolvedPath))
-                    {
-                        FileInfo fInfo = new FileInfo(resolvedPath);
-                        if ((fInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
-                        {
-                            // remember to reset the read-only attribute later
-                            readOnlyFileInfo = fInfo;
-                            // Clear the read-only attribute
-                            fInfo.Attributes &= ~(FileAttributes.ReadOnly);
-                        }
-                    }
-                }
-
-                // if the user knows what he/she is doing and uses "-Force" switch,
-                // then we let more than 1 process write to the same file at the same time
-                FileShare fileShare = Force ? FileShare.ReadWrite : FileShare.Read;
-
-                // mode is controlled by force and ShouldContinue()
-                fileStream = new FileStream(resolvedPath, mode, FileAccess.Write, fileShare);
-
-                // create stream writer
-                // NTRAID#Windows Out Of Band Releases-931008-2006/03/27
-                // For some reason, calling this without specifying
-                // the encoding is different from passing Encoding.Default.
-                if (defaultEncoding)
-                    streamWriter = new StreamWriter(fileStream);
-                else
-                    streamWriter = new StreamWriter(fileStream, resolvedEncoding);
+                MasterStreamOpenImpl(
+                    resolvedPath,
+                    resolvedEncoding,
+                    defaultEncoding,
+                    Append,
+                    Force,
+                    NoClobber,
+                    out fileStream,
+                    out streamWriter,
+                    out readOnlyFileInfo);
             }
             // These are the known exceptions for File.Load and StreamWriter.ctor
             catch (ArgumentException e)
@@ -144,7 +109,7 @@ namespace System.Management.Automation
                 if (NoClobber && File.Exists(resolvedPath))
                 {
                     // This probably happened because the file already exists
-                    ErrorRecord errorRecord = new ErrorRecord(
+                    ErrorRecord errorRecord = new(
                         e, "NoClobber", ErrorCategory.ResourceExists, resolvedPath);
                     errorRecord.ErrorDetails = new ErrorDetails(
                         cmdlet,
@@ -176,6 +141,177 @@ namespace System.Management.Automation
             }
         }
 
+        /// <summary>
+        /// THE method for opening a file for writing.
+        /// Should be used by all cmdlets that write to a file.
+        /// </summary>
+        /// <param name="filePath">Path to the file (as specified on the command line - this method will resolve the path).</param>
+        /// <param name="resolvedEncoding">Encoding (this method will convert the command line string to an Encoding instance).</param>
+        /// <param name="defaultEncoding">If <see langword="true"/>, then we will use default .NET encoding instead of the encoding specified in <paramref name="encoding"/> parameter.</param>
+        /// <param name="Append"></param>
+        /// <param name="Force"></param>
+        /// <param name="NoClobber"></param>
+        /// <param name="fileStream">Result1: <see cref="FileStream"/> opened for writing.</param>
+        /// <param name="streamWriter">Result2: <see cref="StreamWriter"/> (inherits from <see cref="TextWriter"/>) opened for writing.</param>
+        /// <param name="readOnlyFileInfo">Result3: file info that should be used to restore file attributes after done with the file (<see langword="null"/> is this is not needed).</param>
+        /// <param name="isLiteralPath">True if wildcard expansion should be bypassed.</param>
+        internal static void MasterStreamOpen(
+            string filePath,
+            Encoding resolvedEncoding,
+            bool defaultEncoding,
+            bool Append,
+            bool Force,
+            bool NoClobber,
+            out FileStream fileStream,
+            out StreamWriter streamWriter,
+            out FileInfo readOnlyFileInfo,
+            bool isLiteralPath)
+        {
+            fileStream = null;
+            streamWriter = null;
+            readOnlyFileInfo = null;
+            string resolvedPath = ResolveFilePath(filePath, isLiteralPath);
+            try
+            {
+                MasterStreamOpenImpl(
+                    resolvedPath,
+                    resolvedEncoding,
+                    defaultEncoding,
+                    Append,
+                    Force,
+                    NoClobber,
+                    out fileStream,
+                    out streamWriter,
+                    out readOnlyFileInfo);
+            }
+            // These are the known exceptions for File.Load and StreamWriter.ctor
+            catch (ArgumentException e)
+            {
+                AddFileOpenErrorRecord(e);
+                throw;
+            }
+            catch (IOException e)
+            {
+                if (NoClobber && File.Exists(resolvedPath))
+                {
+                    string msg = StringUtil.Format(
+                        PathUtilsStrings.UtilityFileExistsNoClobber,
+                        filePath,
+                        "NoClobber");
+
+                    // This probably happened because the file already exists
+                    ErrorRecord errorRecord = new ErrorRecord(
+                        e, "NoClobber", ErrorCategory.ResourceExists, resolvedPath);
+                    errorRecord.ErrorDetails = new ErrorDetails(msg);
+
+                    e.Data[typeof(ErrorRecord)] = errorRecord;
+                    throw;
+                }
+
+                AddFileOpenErrorRecord(e);
+                throw;
+            }
+            catch (UnauthorizedAccessException e)
+            {
+                AddFileOpenErrorRecord(e);
+                throw;
+            }
+            catch (NotSupportedException e)
+            {
+                AddFileOpenErrorRecord(e);
+                throw;
+            }
+            catch (System.Security.SecurityException e)
+            {
+                AddFileOpenErrorRecord(e);
+                throw;
+            }
+
+            static void AddFileOpenErrorRecord(Exception e)
+            {
+                ErrorRecord errorRecord = new ErrorRecord(
+                    e,
+                    "FileOpenFailure",
+                    ErrorCategory.OpenError,
+                    null);
+
+                e.Data[typeof(ErrorRecord)] = errorRecord;
+            }
+        }
+
+        /// <summary>
+        /// THE method for opening a file for writing.
+        /// Should be used by all cmdlets that write to a file.
+        /// </summary>
+        /// <param name="resolvedPath">Path to the file (as specified on the command line - this method will resolve the path).</param>
+        /// <param name="resolvedEncoding">Encoding (this method will convert the command line string to an Encoding instance).</param>
+        /// <param name="defaultEncoding">If <see langword="true"/>, then we will use default .NET encoding instead of the encoding specified in <paramref name="encoding"/> parameter.</param>
+        /// <param name="Append"></param>
+        /// <param name="Force"></param>
+        /// <param name="NoClobber"></param>
+        /// <param name="fileStream">Result1: <see cref="FileStream"/> opened for writing.</param>
+        /// <param name="streamWriter">Result2: <see cref="StreamWriter"/> (inherits from <see cref="TextWriter"/>) opened for writing.</param>
+        /// <param name="readOnlyFileInfo">Result3: file info that should be used to restore file attributes after done with the file (<see langword="null"/> is this is not needed).</param>
+        internal static void MasterStreamOpenImpl(
+            string resolvedPath,
+            Encoding resolvedEncoding,
+            bool defaultEncoding,
+            bool Append,
+            bool Force,
+            bool NoClobber,
+            out FileStream fileStream,
+            out StreamWriter streamWriter,
+            out FileInfo readOnlyFileInfo)
+        {
+            fileStream = null;
+            streamWriter = null;
+            readOnlyFileInfo = null;
+
+            // variable to track file open mode
+            // this is controlled by append/force parameters
+            FileMode mode = FileMode.Create;
+            if (Append)
+            {
+                mode = FileMode.Append;
+            }
+            else if (NoClobber)
+            {
+                // throw IOException if file exists
+                mode = FileMode.CreateNew;
+            }
+
+            if (Force && (Append || !NoClobber))
+            {
+                if (File.Exists(resolvedPath))
+                {
+                    FileInfo fInfo = new FileInfo(resolvedPath);
+                    if ((fInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
+                    {
+                        // remember to reset the read-only attribute later
+                        readOnlyFileInfo = fInfo;
+                        // Clear the read-only attribute
+                        fInfo.Attributes &= ~(FileAttributes.ReadOnly);
+                    }
+                }
+            }
+
+            // if the user knows what he/she is doing and uses "-Force" switch,
+            // then we let more than 1 process write to the same file at the same time
+            FileShare fileShare = Force ? FileShare.ReadWrite : FileShare.Read;
+
+            // mode is controlled by force and ShouldContinue()
+            fileStream = new FileStream(resolvedPath, mode, FileAccess.Write, fileShare);
+
+            // create stream writer
+            // NTRAID#Windows Out Of Band Releases-931008-2006/03/27
+            // For some reason, calling this without specifying
+            // the encoding is different from passing Encoding.Default.
+            if (defaultEncoding)
+                streamWriter = new StreamWriter(fileStream);
+            else
+                streamWriter = new StreamWriter(fileStream, resolvedEncoding);
+        }
+
         internal static void ReportFileOpenFailure(Cmdlet cmdlet, string filePath, Exception e)
         {
             ErrorRecord errorRecord = new ErrorRecord(
@@ -187,6 +323,20 @@ namespace System.Management.Automation
             cmdlet.ThrowTerminatingError(errorRecord);
         }
 
+        internal static void ReportFileOpenFailure(string filePath, Exception e)
+        {
+            ErrorRecord errorRecord = new ErrorRecord(
+                e,
+                "FileOpenFailure",
+                ErrorCategory.OpenError,
+                null);
+
+            throw new RuntimeException(
+                e.Message,
+                errorRecord.Exception,
+                errorRecord);
+        }
+
         internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, Encoding encoding, bool isLiteralPath)
         {
             FileStream fileStream = OpenFileStream(filePath, command, isLiteralPath);
@@ -308,6 +458,63 @@ namespace System.Management.Automation
             return path;
         }
 
+        /// <summary>
+        /// Resolve a user provided file name or path (including globbing characters)
+        /// to a fully qualified file path, using the file system provider.
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <param name="isLiteralPath"></param>
+        /// <returns></returns>
+        internal static string ResolveFilePath(string filePath, bool isLiteralPath)
+        {
+            string path = null;
+
+            SessionState sessionState = LocalPipeline.GetExecutionContextFromTLS()?.EngineSessionState?.PublicSessionState;
+            if (sessionState is null)
+            {
+                return null;
+            }
+
+            try
+            {
+                ProviderInfo provider = null;
+                PSDriveInfo drive = null;
+                List<string> filePaths = new();
+
+                if (isLiteralPath)
+                {
+                    filePaths.Add(sessionState.Path.GetUnresolvedProviderPathFromPSPath(filePath, out provider, out drive));
+                }
+                else
+                {
+                    filePaths.AddRange(sessionState.Path.GetResolvedProviderPathFromPSPath(filePath, out provider));
+                }
+
+                if (!provider.NameEquals(FileSystemProvider.ProviderName))
+                {
+                    ReportWrongProviderType(provider.FullName);
+                }
+
+                if (filePaths.Count > 1)
+                {
+                    ReportMultipleFilesNotSupported();
+                }
+
+                if (filePaths.Count == 0)
+                {
+                    ReportWildcardingFailure(filePath);
+                }
+
+                path = filePaths[0];
+            }
+            catch (ItemNotFoundException)
+            {
+                path = null;
+            }
+
+            return path;
+        }
+
         internal static void ReportWrongProviderType(Cmdlet cmdlet, string providerId)
         {
             string msg = StringUtil.Format(PathUtilsStrings.OutFile_ReadWriteFileNotFileSystemProvider, providerId);
@@ -322,6 +529,23 @@ namespace System.Management.Automation
             cmdlet.ThrowTerminatingError(errorRecord);
         }
 
+        internal static void ReportWrongProviderType(string providerId)
+        {
+            string msg = StringUtil.Format(PathUtilsStrings.OutFile_ReadWriteFileNotFileSystemProvider, providerId);
+
+            PSInvalidOperationException exception = PSTraceSource.NewInvalidOperationException();
+
+            ErrorRecord errorRecord = new(
+                exception,
+                "ReadWriteFileNotFileSystemProvider",
+                ErrorCategory.InvalidArgument,
+                null);
+
+            errorRecord.ErrorDetails = new ErrorDetails(msg);
+            exception.Data[typeof(ErrorRecord)] = errorRecord;
+            throw exception;
+        }
+
         internal static void ReportMultipleFilesNotSupported(Cmdlet cmdlet)
         {
             string msg = StringUtil.Format(PathUtilsStrings.OutFile_MultipleFilesNotSupported);
@@ -336,6 +560,23 @@ namespace System.Management.Automation
             cmdlet.ThrowTerminatingError(errorRecord);
         }
 
+        internal static void ReportMultipleFilesNotSupported()
+        {
+            string msg = StringUtil.Format(PathUtilsStrings.OutFile_MultipleFilesNotSupported);
+
+            PSInvalidOperationException exception = PSTraceSource.NewInvalidOperationException();
+
+            ErrorRecord errorRecord = new(
+                exception,
+                "ReadWriteMultipleFilesNotSupported",
+                ErrorCategory.InvalidArgument,
+                null);
+
+            errorRecord.ErrorDetails = new ErrorDetails(msg);
+            exception.Data[typeof(ErrorRecord)] = errorRecord;
+            throw exception;
+        }
+
         internal static void ReportWildcardingFailure(Cmdlet cmdlet, string filePath)
         {
             string msg = StringUtil.Format(PathUtilsStrings.OutFile_DidNotResolveFile, filePath);
@@ -350,6 +591,22 @@ namespace System.Management.Automation
             cmdlet.ThrowTerminatingError(errorRecord);
         }
 
+        internal static void ReportWildcardingFailure(string filePath)
+        {
+            string msg = StringUtil.Format(PathUtilsStrings.OutFile_DidNotResolveFile, filePath);
+
+            FileNotFoundException exception = new();
+            ErrorRecord errorRecord = new(
+                exception,
+                "FileOpenFailure",
+                ErrorCategory.OpenError,
+                filePath);
+
+            errorRecord.ErrorDetails = new ErrorDetails(msg);
+            exception.Data[typeof(ErrorRecord)] = errorRecord;
+            throw exception;
+        }
+
         internal static DirectoryInfo CreateModuleDirectory(PSCmdlet cmdlet, string moduleNameOrPath, bool force)
         {
             Dbg.Assert(cmdlet != null, "Caller should verify cmdlet != null");

test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1

@@ -0,0 +1,66 @@
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Functional tests to verify that output from native executables is not encoded
+# and decoded when piping to another native executable.
+
+Describe 'Native command byte piping tests' -Tags 'CI' {
+    BeforeAll {
+        $originalDefaultParameterValues = $PSDefaultParameterValues.Clone()
+        if (-not [ExperimentalFeature]::IsEnabled('PSNativeCommandPreserveBytePipe'))
+        {
+            $PSDefaultParameterValues['It:Skip'] = $true
+            return
+        }
+
+        # Without this the test would otherwise be hard coded to a specific set
+        # of [Console]::OutputEncoding/$OutputEncoding settings.
+        $mangledFFByte = $OutputEncoding.GetBytes(
+            [Console]::OutputEncoding.GetString(0xFF) + [Environment]::NewLine).
+            ForEach{ '{0:X2}' -f [int]$_ }
+    }
+
+    AfterAll {
+        $global:PSDefaultParameterValues = $originalDefaultParameterValues
+    }
+
+    It 'Bytes are retained between native executables' {
+        testexe -writebytes FF | testexe -readbytes | Should -BeExactly FF
+    }
+
+    It 'Byte literals are retained when piped directly' {
+        0xFFuy | testexe -readbytes | Should -BeExactly FF
+        0xBEuy, 0xEFuy | testexe -readbytes | Should -BeExactly BE, EF
+        ,[byte[]](0xBEuy, 0xEFuy) | testexe -readbytes | Should -BeExactly BE, EF
+    }
+
+    It 'Output behavior falls back when stderr is redirected to stdout' {
+        testexe -writebytes FF 2>&1 | testexe -readbytes | Should -BeExactly $mangledFFByte
+    }
+
+    It 'Bytes are retained when using SMA.PowerShell' {
+        $ps = $null
+        try {
+            $ps = [powershell]::Create().
+                AddCommand('testexe').AddArgument('-writebytes').AddArgument('FF').
+                AddCommand('testexe').AddArgument('-readbytes')
+
+            $ps.Invoke() | Should -BeExactly 'FF'
+        } finally {
+            ($ps)?.Dispose()
+        }
+    }
+
+    It 'Bytes are retained when using SteppablePipeline' {
+        $pipe = $null
+        try {
+            $pipe = { testexe -writebytes FF | testexe -readbytes }.GetSteppablePipeline('Internal')
+            $pipe.Begin($false)
+            $pipe.Process()
+            $pipe.End() | Should -BeExactly 'FF'
+        } finally {
+            ($pipe)?.Dispose()
+        }
+    }
+}

test/tools/TestExe/TestExe.cs

@@ -4,7 +4,10 @@
 using System;
 using System.Threading;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.IO;
+using System.Globalization;
 
 namespace TestExe
 {
@@ -33,6 +36,12 @@ namespace TestExe
                     case "-stderr":
                         Console.Error.WriteLine(args[1]);
                         break;
+                    case "-readbytes":
+                        ReadBytes();
+                        break;
+                    case "-writebytes":
+                        WriteBytes(args.AsSpan()[1..]);
+                        break;
                     case "--help":
                     case "-h":
                         PrintHelp();
@@ -52,6 +61,45 @@ namespace TestExe
             return exitCode;
         }
 
+        private static void WriteBytes(ReadOnlySpan<string> args)
+        {
+            using Stream stdout = Console.OpenStandardOutput();
+            foreach (string arg in args)
+            {
+                if (!byte.TryParse(arg, NumberStyles.AllowHexSpecifier, provider: null, out byte value))
+                {
+                    throw new ArgumentException(
+                        nameof(args),
+                        "All args after -writebytes must be single byte hex strings.");
+                }
+
+                stdout.WriteByte(value);
+            }
+        }
+
+        [SkipLocalsInit]
+        private static void ReadBytes()
+        {
+            using Stream stdin = Console.OpenStandardInput();
+            Span<byte> buffer = stackalloc byte[0x200];
+            Unsafe.InitBlock(ref MemoryMarshal.GetReference(buffer), 0, 0x200);
+            Span<char> hex = stackalloc char[] { '\0', '\0' };
+            while (true)
+            {
+                int received = stdin.Read(buffer);
+                if (received is 0)
+                {
+                    return;
+                }
+
+                for (int i = 0; i < received; i++)
+                {
+                    buffer[i].TryFormat(hex, out _, "X2");
+                    Console.Out.WriteLine(hex);
+                }
+            }
+        }
+
         // <Summary>
         // Echos back to stdout the arguments passed in
         // </Summary>