From 2424ad83aa4d44fe9e8f507485744e00c66cde58 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Thu, 27 Apr 2023 20:17:32 -0400 Subject: [PATCH] Support byte stream piping between native commands and file redirection (#17857) --- .../engine/AsyncByteStreamTransfer.cs | 83 +++++ .../engine/BytePipe.cs | 115 ++++++ .../ExperimentalFeature.cs | 4 + .../engine/NativeCommandProcessor.cs | 222 +++++++++-- .../engine/pipeline.cs | 31 ++ .../engine/runtime/Operations/MiscOps.cs | 37 +- .../utils/PathUtils.cs | 351 +++++++++++++++--- .../Basic/NativeCommandBytePiping.Tests.ps1 | 66 ++++ test/tools/TestExe/TestExe.cs | 48 +++ 9 files changed, 878 insertions(+), 79 deletions(-) create mode 100644 src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs create mode 100644 src/System.Management.Automation/engine/BytePipe.cs create mode 100644 test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1 diff --git a/src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs b/src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs new file mode 100644 index 0000000000..f1391924d6 --- /dev/null +++ b/src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#nullable enable + +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Management.Automation; + +/// +/// Represents the transfer of bytes from one to another +/// asynchronously. +/// +internal sealed class AsyncByteStreamTransfer : IDisposable +{ + private const int DefaultBufferSize = 1024; + + private readonly BytePipe _bytePipe; + + private readonly BytePipe _destinationPipe; + + private readonly Memory _buffer; + + private readonly CancellationTokenSource _cts = new(); + + private Task? _readToBufferTask; + + public AsyncByteStreamTransfer( + BytePipe bytePipe, + BytePipe destinationPipe) + { + _bytePipe = bytePipe; + _destinationPipe = destinationPipe; + _buffer = new byte[DefaultBufferSize]; + } + + public Task EOF => _readToBufferTask ?? Task.CompletedTask; + + public void BeginReadChunks() + { + _readToBufferTask = Task.Run(ReadBufferAsync); + } + + public void Dispose() => _cts.Cancel(); + + private async Task ReadBufferAsync() + { + Stream stream; + Stream? destinationStream = null; + try + { + stream = await _bytePipe.GetStream(_cts.Token); + destinationStream = await _destinationPipe.GetStream(_cts.Token); + + while (true) + { + int bytesRead; + bytesRead = await stream.ReadAsync(_buffer, _cts.Token); + if (bytesRead is 0) + { + break; + } + + destinationStream.Write(_buffer.Span.Slice(0, bytesRead)); + } + } + catch (IOException) + { + return; + } + catch (OperationCanceledException) + { + return; + } + finally + { + destinationStream?.Close(); + } + } +} diff --git a/src/System.Management.Automation/engine/BytePipe.cs b/src/System.Management.Automation/engine/BytePipe.cs new file mode 100644 index 0000000000..3d4832d604 --- /dev/null +++ b/src/System.Management.Automation/engine/BytePipe.cs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#nullable enable + +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerShell.Telemetry; + +namespace System.Management.Automation; + +/// +/// Represents a lazily retrieved for transfering bytes +/// to or from. +/// +internal abstract class BytePipe +{ + public abstract Task GetStream(CancellationToken cancellationToken); + + internal AsyncByteStreamTransfer Bind(BytePipe bytePipe) + { + Debug.Assert(bytePipe is not null); + return new AsyncByteStreamTransfer(bytePipe, destinationPipe: this); + } +} + +/// +/// Represents a lazily retrieved from the underlying +/// . +/// +internal sealed class NativeCommandProcessorBytePipe : BytePipe +{ + private readonly NativeCommandProcessor _nativeCommand; + + private readonly bool _stdout; + + internal NativeCommandProcessorBytePipe( + NativeCommandProcessor nativeCommand, + bool stdout) + { + Debug.Assert(nativeCommand is not null); + _nativeCommand = nativeCommand; + _stdout = stdout; + } + + public override async Task GetStream(CancellationToken cancellationToken) + { + // If the native command we're wrapping is the upstream command then + // NativeCommandProcessor.Prepare will have already been called before + // the creation of this BytePipe. + if (_stdout) + { + return _nativeCommand.GetStream(stdout: true); + } + + await _nativeCommand.WaitForProcessInitializationAsync(cancellationToken); + return _nativeCommand.GetStream(stdout: false); + } +} + +/// +/// Provides an byte pipe implementation representing a . +/// +internal sealed class FileBytePipe : BytePipe +{ + private readonly Stream _stream; + + private FileBytePipe(Stream stream) + { + Debug.Assert(stream is not null); + _stream = stream; + } + + internal static FileBytePipe Create(string fileName, bool append) + { + FileStream fileStream; + try + { + PathUtils.MasterStreamOpen( + fileName, + resolvedEncoding: null, + defaultEncoding: false, + append, + Force: true, + NoClobber: false, + out fileStream, + streamWriter: out _, + readOnlyFileInfo: out _, + isLiteralPath: true); + } + catch (Exception e) when (e.Data.Contains(typeof(ErrorRecord))) + { + // The error record is attached to the exception when thrown to preserve + // the call stack. + ErrorRecord? errorRecord = e.Data[typeof(ErrorRecord)] as ErrorRecord; + if (errorRecord is null) + { + throw; + } + + e.Data.Remove(typeof(ErrorRecord)); + throw new RuntimeException(null, e, errorRecord); + } + + ApplicationInsightsTelemetry.SendExperimentalUseData( + ExperimentalFeature.PSNativeCommandPreserveBytePipe, + "f"); + + return new FileBytePipe(fileStream); + } + + public override Task GetStream(CancellationToken cancellationToken) => Task.FromResult(_stream); +} diff --git a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs index f7c2a73ba2..bc826fa278 100644 --- a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs +++ b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs @@ -22,6 +22,7 @@ namespace System.Management.Automation internal const string EngineSource = "PSEngine"; internal const string PSNativeCommandErrorActionPreferenceFeatureName = "PSNativeCommandErrorActionPreference"; + internal const string PSNativeCommandPreserveBytePipe = "PSNativeCommandPreserveBytePipe"; internal const string PSModuleAutoLoadSkipOfflineFilesFeatureName = "PSModuleAutoLoadSkipOfflineFiles"; internal const string PSCustomTableHeaderLabelDecoration = "PSCustomTableHeaderLabelDecoration"; internal const string PSFeedbackProvider = "PSFeedbackProvider"; @@ -126,6 +127,9 @@ namespace System.Management.Automation new ExperimentalFeature( name: PSCustomTableHeaderLabelDecoration, description: "Formatting differentiation for table header labels that aren't property members"), + new ExperimentalFeature( + name: PSNativeCommandPreserveBytePipe, + description: "Byte output is retained when piping between two or more native commands"), new ExperimentalFeature( name: PSFeedbackProvider, description: "Replace the hard-coded suggestion framework with the extensible feedback provider"), diff --git a/src/System.Management.Automation/engine/NativeCommandProcessor.cs b/src/System.Management.Automation/engine/NativeCommandProcessor.cs index f34fc1987b..3bb0fef21b 100644 --- a/src/System.Management.Automation/engine/NativeCommandProcessor.cs +++ b/src/System.Management.Automation/engine/NativeCommandProcessor.cs @@ -17,6 +17,7 @@ using System.Runtime.InteropServices; using System.Runtime.Serialization; using System.Text; using System.Threading; +using System.Threading.Tasks; using System.Xml; using Microsoft.PowerShell.Telemetry; using Dbg = System.Management.Automation.Diagnostics; @@ -339,16 +340,16 @@ namespace System.Management.Automation } } + internal NativeCommandProcessor DownStreamNativeCommand { get; set; } + + internal bool UpstreamIsNativeCommand { get; set; } + + internal BytePipe StdOutDestination { get; set; } + #endregion ctor/native command properties #region parameter binder - /// - /// Variable which is set to true when prepare is called. - /// Parameter Binder should only be created after Prepare method is called. - /// - private bool _isPreparedCalled = false; - /// /// Parameter binder used by this command processor. /// @@ -365,8 +366,6 @@ namespace System.Management.Automation /// internal ParameterBinderController NewParameterBinderController(InternalCommand command) { - Dbg.Assert(_isPreparedCalled, "parameter binder should not be created before prepared is called"); - if (_isMiniShell) { _nativeParameterBinderController = @@ -405,8 +404,6 @@ namespace System.Management.Automation /// internal override void Prepare(IDictionary psDefaultParameterValues) { - _isPreparedCalled = true; - // Check if the application is minishell _isMiniShell = IsMiniShell(); @@ -436,9 +433,15 @@ namespace System.Management.Automation { try { - while (Read()) + // If upstream is a native command it'll be writing directly to our stdin stream + // so we can skip reading here. + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + && !UpstreamIsNativeCommand) { - _inputWriter.Add(Command.CurrentPipelineObject); + while (Read()) + { + _inputWriter.Add(Command.CurrentPipelineObject); + } } ConsumeAvailableNativeProcessOutput(blocking: false); @@ -496,6 +499,59 @@ namespace System.Management.Automation /// private readonly object _sync = new object(); + private SemaphoreSlim _processInitialized; + + internal async Task WaitForProcessInitializationAsync(CancellationToken cancellationToken) + { + SemaphoreSlim processInitialized = _processInitialized; + if (processInitialized is null) + { + lock (_sync) + { + processInitialized = _processInitialized ??= new SemaphoreSlim(0, 1); + } + } + + try + { + await processInitialized.WaitAsync(cancellationToken); + } + finally + { + processInitialized.Release(); + } + } + + /// + /// Creates a pipe representing the streaming of unprocessed bytes. + /// + /// + /// The stream that the pipe should represent. + /// for stdout, for stdin. + /// + /// A new byte pipe representing the specified stream. + internal BytePipe CreateBytePipe(bool stdout) => new NativeCommandProcessorBytePipe(this, stdout); + + /// + /// Gets the specified base for the underlying + /// . + /// + /// + /// The stream that should be retrieved. for + /// stdout, for stdin. + /// + /// The specified . + internal Stream GetStream(bool stdout) + { + Debug.Assert( + _nativeProcess is not null, + "Caller should verify that initialization has completed before attempting to get the underlying stream."); + + return stdout + ? _nativeProcess.StandardOutput.BaseStream + : _nativeProcess.StandardInput.BaseStream; + } + /// /// Executes the native command once all of the input has been gathered. /// @@ -581,6 +637,19 @@ namespace System.Management.Automation { _nativeProcess = new Process() { StartInfo = startInfo }; _nativeProcess.Start(); + if (UpstreamIsNativeCommand) + { + SemaphoreSlim processInitialized = _processInitialized; + if (processInitialized is null) + { + lock (_sync) + { + processInitialized = _processInitialized ??= new SemaphoreSlim(0, 1); + } + } + + processInitialized?.Release(); + } } catch (Win32Exception) { @@ -691,7 +760,9 @@ namespace System.Management.Automation lock (_sync) { - if (!_stopped) + if (!_stopped + && (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + || !UpstreamIsNativeCommand)) { _inputWriter.Start(_nativeProcess, inputFormat); } @@ -740,6 +811,8 @@ namespace System.Management.Automation } } + private AsyncByteStreamTransfer _stdOutByteTransfer; + private void InitOutputQueue() { // if output is redirected, start reading output of process in queue. @@ -749,9 +822,38 @@ namespace System.Management.Automation { if (!_stopped) { + if (CommandRuntime.ErrorMergeTo is MshCommandRuntime.MergeDataStream.Output) + { + StdOutDestination = null; + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)) + { + if (DownStreamNativeCommand is not null) + { + DownStreamNativeCommand.UpstreamIsNativeCommand = false; + DownStreamNativeCommand = null; + } + } + } + _nativeProcessOutputQueue = new BlockingCollection(); // we don't assign the handler to anything, because it's used only for objects marshaling - new ProcessOutputHandler(_nativeProcess, _nativeProcessOutputQueue); + BytePipe stdOutDestination = ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + ? StdOutDestination ?? DownStreamNativeCommand?.CreateBytePipe(stdout: false) + : null; + + BytePipe stdOutSource = null; + if (stdOutDestination is not null) + { + stdOutSource = CreateBytePipe(stdout: true); + } + + _ = new ProcessOutputHandler( + _nativeProcess, + _nativeProcessOutputQueue, + stdOutDestination, + stdOutSource, + out _stdOutByteTransfer); + } } } @@ -761,6 +863,13 @@ namespace System.Management.Automation { if (blocking) { + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + && _stdOutByteTransfer is not null) + { + _stdOutByteTransfer.EOF.GetAwaiter().GetResult(); + return null; + } + // If adding was completed and collection is empty (IsCompleted == true) // there is no need to do a blocking Take(), we should just return. if (!_nativeProcessOutputQueue.IsCompleted) @@ -784,6 +893,12 @@ namespace System.Management.Automation } else { + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + && _stdOutByteTransfer is not null) + { + return null; + } + ProcessOutputObject record = null; _nativeProcessOutputQueue.TryTake(out record); return record; @@ -822,7 +937,11 @@ namespace System.Management.Automation if (!_isRunningInBackground) { // Wait for input writer to finish. - _inputWriter.Done(); + if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + || !UpstreamIsNativeCommand) + { + _inputWriter.Done(); + } // read all the available output in the blocking way ConsumeAvailableNativeProcessOutput(blocking: true); @@ -1188,8 +1307,13 @@ namespace System.Management.Automation if (!_runStandAlone) { // Stop input writer - _inputWriter.Stop(); + if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + || !UpstreamIsNativeCommand) + { + _inputWriter.Stop(); + } + _stdOutByteTransfer?.Dispose(); KillProcess(_nativeProcess); } } @@ -1675,7 +1799,19 @@ namespace System.Management.Automation private bool _isXmlCliError; private readonly string _processFileName; + private readonly AsyncByteStreamTransfer _stdOutDrainer; + public ProcessOutputHandler(Process process, BlockingCollection queue) + : this(process, queue, null, null, out _) + { + } + + public ProcessOutputHandler( + Process process, + BlockingCollection queue, + BytePipe stdOutDestination, + BytePipe stdOutSource, + out AsyncByteStreamTransfer stdOutDrainer) { Debug.Assert(process.StartInfo.RedirectStandardOutput || process.StartInfo.RedirectStandardError, "Caller should redirect at least one stream"); _refCount = 0; @@ -1695,14 +1831,6 @@ namespace System.Management.Automation } // once we have _refCount, we can start processing - if (process.StartInfo.RedirectStandardOutput) - { - _isFirstOutput = true; - _isXmlCliOutput = false; - process.OutputDataReceived += OutputHandler; - process.BeginOutputReadLine(); - } - if (process.StartInfo.RedirectStandardError) { _isFirstError = true; @@ -1710,6 +1838,26 @@ namespace System.Management.Automation process.ErrorDataReceived += ErrorHandler; process.BeginErrorReadLine(); } + + stdOutDrainer = null; + + if (!process.StartInfo.RedirectStandardOutput) + { + return; + } + + if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + || stdOutDestination is null) + { + _isFirstOutput = true; + _isXmlCliOutput = false; + process.OutputDataReceived += OutputHandler; + process.BeginOutputReadLine(); + return; + } + + stdOutDrainer = _stdOutDrainer = stdOutDestination.Bind(stdOutSource); + stdOutDrainer.BeginReadChunks(); } private void decrementRefCount() @@ -1948,14 +2096,30 @@ namespace System.Management.Automation return; } - if (_inputFormat == NativeCommandIOFormat.Text) - { - AddTextInput(input); - } - else // Xml + if (_inputFormat is not NativeCommandIOFormat.Text) { AddXmlInput(input); + return; } + + object baseObjInput = PSObject.Base(input); + + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)) + { + if (baseObjInput is byte[] bytes) + { + _streamWriter.BaseStream.Write(bytes, 0, bytes.Length); + return; + } + + if (baseObjInput is byte b) + { + _streamWriter.BaseStream.WriteByte(b); + return; + } + } + + AddTextInput(input); } private void AddTextInput(object input) diff --git a/src/System.Management.Automation/engine/pipeline.cs b/src/System.Management.Automation/engine/pipeline.cs index dba038f91b..a045a4f83f 100644 --- a/src/System.Management.Automation/engine/pipeline.cs +++ b/src/System.Management.Automation/engine/pipeline.cs @@ -43,6 +43,10 @@ namespace System.Management.Automation.Internal private bool _linkedSuccessOutput = false; private bool _linkedErrorOutput = false; + private NativeCommandProcessor _lastNativeCommand; + + private bool _haveReportedNativePipeUsage; + #if !CORECLR // Impersonation Not Supported On CSS // This is the security context when the pipeline was allocated internal System.Security.SecurityContext SecurityContext = @@ -257,6 +261,33 @@ namespace System.Management.Automation.Internal /// internal int Add(CommandProcessorBase commandProcessor) { + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)) + { + if (commandProcessor is NativeCommandProcessor nativeCommand) + { + if (_lastNativeCommand is not null) + { + // Only report experimental feature usage once per pipeline. + if (!_haveReportedNativePipeUsage) + { + ApplicationInsightsTelemetry.SendExperimentalUseData( + ExperimentalFeature.PSNativeCommandPreserveBytePipe, + "p"); + _haveReportedNativePipeUsage = true; + } + + _lastNativeCommand.DownStreamNativeCommand = nativeCommand; + nativeCommand.UpstreamIsNativeCommand = true; + } + + _lastNativeCommand = nativeCommand; + } + else + { + _lastNativeCommand = null; + } + } + commandProcessor.CommandRuntime.PipelineProcessor = this; return AddCommand(commandProcessor, _commands.Count, readErrorQueue: false); } diff --git a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs index d37a2b4945..152fb28914 100644 --- a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs +++ b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs @@ -162,6 +162,7 @@ namespace System.Management.Automation (cmd is ScriptCommand || cmd is PSScriptCmdlet); bool isNativeCommand = commandProcessor is NativeCommandProcessor; + for (int i = commandIndex + 1; i < commandElements.Length; ++i) { var cpi = commandElements[i]; @@ -208,9 +209,27 @@ namespace System.Management.Automation bool redirectedInformation = false; if (redirections != null) { - foreach (var redirection in redirections) + bool shouldProcessMergesFirst = ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe) + && isNativeCommand; + + if (shouldProcessMergesFirst) { - redirection.Bind(pipe, commandProcessor, context); + foreach (CommandRedirection redirection in redirections) + { + if (redirection is MergingRedirection) + { + redirection.Bind(pipe, commandProcessor, context); + } + } + } + + foreach (CommandRedirection redirection in redirections) + { + if (!shouldProcessMergesFirst || redirection is not MergingRedirection) + { + redirection.Bind(pipe, commandProcessor, context); + } + switch (redirection.FromStream) { case RedirectionStream.Error: @@ -1050,6 +1069,18 @@ namespace System.Management.Automation // dir > out internal override void Bind(PipelineProcessor pipelineProcessor, CommandProcessorBase commandProcessor, ExecutionContext context) { + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandPreserveBytePipe)) + { + if (commandProcessor is NativeCommandProcessor nativeCommand + && nativeCommand.CommandRuntime.ErrorMergeTo is not MshCommandRuntime.MergeDataStream.Output + && FromStream is RedirectionStream.Output + && !string.IsNullOrWhiteSpace(File)) + { + nativeCommand.StdOutDestination = FileBytePipe.Create(File, Appending); + return; + } + } + Pipe pipe = GetRedirectionPipe(context, pipelineProcessor); switch (FromStream) @@ -3627,7 +3658,7 @@ namespace System.Management.Automation } catch (PSSecurityException) { - // ReportContent() will throw PSSecurityException if AMSI detects malware, which + // ReportContent() will throw PSSecurityException if AMSI detects malware, which // must be propagated. throw; } diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs index 558ca25f2f..1d7c3140d6 100644 --- a/src/System.Management.Automation/utils/PathUtils.cs +++ b/src/System.Management.Automation/utils/PathUtils.cs @@ -6,6 +6,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Management.Automation.Internal; +using System.Management.Automation.Runspaces; using System.Runtime.CompilerServices; using System.Text; using Microsoft.PowerShell.Commands; @@ -83,55 +84,19 @@ namespace System.Management.Automation fileStream = null; streamWriter = null; readOnlyFileInfo = null; - - // resolve the path and the encoding string resolvedPath = ResolveFilePath(filePath, cmdlet, isLiteralPath); - try { - // variable to track file open mode - // this is controlled by append/force parameters - FileMode mode = FileMode.Create; - if (Append) - { - mode = FileMode.Append; - } - else if (NoClobber) - { - // throw IOException if file exists - mode = FileMode.CreateNew; - } - - if (Force && (Append || !NoClobber)) - { - if (File.Exists(resolvedPath)) - { - FileInfo fInfo = new FileInfo(resolvedPath); - if ((fInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) - { - // remember to reset the read-only attribute later - readOnlyFileInfo = fInfo; - // Clear the read-only attribute - fInfo.Attributes &= ~(FileAttributes.ReadOnly); - } - } - } - - // if the user knows what he/she is doing and uses "-Force" switch, - // then we let more than 1 process write to the same file at the same time - FileShare fileShare = Force ? FileShare.ReadWrite : FileShare.Read; - - // mode is controlled by force and ShouldContinue() - fileStream = new FileStream(resolvedPath, mode, FileAccess.Write, fileShare); - - // create stream writer - // NTRAID#Windows Out Of Band Releases-931008-2006/03/27 - // For some reason, calling this without specifying - // the encoding is different from passing Encoding.Default. - if (defaultEncoding) - streamWriter = new StreamWriter(fileStream); - else - streamWriter = new StreamWriter(fileStream, resolvedEncoding); + MasterStreamOpenImpl( + resolvedPath, + resolvedEncoding, + defaultEncoding, + Append, + Force, + NoClobber, + out fileStream, + out streamWriter, + out readOnlyFileInfo); } // These are the known exceptions for File.Load and StreamWriter.ctor catch (ArgumentException e) @@ -144,7 +109,7 @@ namespace System.Management.Automation if (NoClobber && File.Exists(resolvedPath)) { // This probably happened because the file already exists - ErrorRecord errorRecord = new ErrorRecord( + ErrorRecord errorRecord = new( e, "NoClobber", ErrorCategory.ResourceExists, resolvedPath); errorRecord.ErrorDetails = new ErrorDetails( cmdlet, @@ -176,6 +141,177 @@ namespace System.Management.Automation } } + /// + /// THE method for opening a file for writing. + /// Should be used by all cmdlets that write to a file. + /// + /// Path to the file (as specified on the command line - this method will resolve the path). + /// Encoding (this method will convert the command line string to an Encoding instance). + /// If , then we will use default .NET encoding instead of the encoding specified in parameter. + /// + /// + /// + /// Result1: opened for writing. + /// Result2: (inherits from ) opened for writing. + /// Result3: file info that should be used to restore file attributes after done with the file ( is this is not needed). + /// True if wildcard expansion should be bypassed. + internal static void MasterStreamOpen( + string filePath, + Encoding resolvedEncoding, + bool defaultEncoding, + bool Append, + bool Force, + bool NoClobber, + out FileStream fileStream, + out StreamWriter streamWriter, + out FileInfo readOnlyFileInfo, + bool isLiteralPath) + { + fileStream = null; + streamWriter = null; + readOnlyFileInfo = null; + string resolvedPath = ResolveFilePath(filePath, isLiteralPath); + try + { + MasterStreamOpenImpl( + resolvedPath, + resolvedEncoding, + defaultEncoding, + Append, + Force, + NoClobber, + out fileStream, + out streamWriter, + out readOnlyFileInfo); + } + // These are the known exceptions for File.Load and StreamWriter.ctor + catch (ArgumentException e) + { + AddFileOpenErrorRecord(e); + throw; + } + catch (IOException e) + { + if (NoClobber && File.Exists(resolvedPath)) + { + string msg = StringUtil.Format( + PathUtilsStrings.UtilityFileExistsNoClobber, + filePath, + "NoClobber"); + + // This probably happened because the file already exists + ErrorRecord errorRecord = new ErrorRecord( + e, "NoClobber", ErrorCategory.ResourceExists, resolvedPath); + errorRecord.ErrorDetails = new ErrorDetails(msg); + + e.Data[typeof(ErrorRecord)] = errorRecord; + throw; + } + + AddFileOpenErrorRecord(e); + throw; + } + catch (UnauthorizedAccessException e) + { + AddFileOpenErrorRecord(e); + throw; + } + catch (NotSupportedException e) + { + AddFileOpenErrorRecord(e); + throw; + } + catch (System.Security.SecurityException e) + { + AddFileOpenErrorRecord(e); + throw; + } + + static void AddFileOpenErrorRecord(Exception e) + { + ErrorRecord errorRecord = new ErrorRecord( + e, + "FileOpenFailure", + ErrorCategory.OpenError, + null); + + e.Data[typeof(ErrorRecord)] = errorRecord; + } + } + + /// + /// THE method for opening a file for writing. + /// Should be used by all cmdlets that write to a file. + /// + /// Path to the file (as specified on the command line - this method will resolve the path). + /// Encoding (this method will convert the command line string to an Encoding instance). + /// If , then we will use default .NET encoding instead of the encoding specified in parameter. + /// + /// + /// + /// Result1: opened for writing. + /// Result2: (inherits from ) opened for writing. + /// Result3: file info that should be used to restore file attributes after done with the file ( is this is not needed). + internal static void MasterStreamOpenImpl( + string resolvedPath, + Encoding resolvedEncoding, + bool defaultEncoding, + bool Append, + bool Force, + bool NoClobber, + out FileStream fileStream, + out StreamWriter streamWriter, + out FileInfo readOnlyFileInfo) + { + fileStream = null; + streamWriter = null; + readOnlyFileInfo = null; + + // variable to track file open mode + // this is controlled by append/force parameters + FileMode mode = FileMode.Create; + if (Append) + { + mode = FileMode.Append; + } + else if (NoClobber) + { + // throw IOException if file exists + mode = FileMode.CreateNew; + } + + if (Force && (Append || !NoClobber)) + { + if (File.Exists(resolvedPath)) + { + FileInfo fInfo = new FileInfo(resolvedPath); + if ((fInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) + { + // remember to reset the read-only attribute later + readOnlyFileInfo = fInfo; + // Clear the read-only attribute + fInfo.Attributes &= ~(FileAttributes.ReadOnly); + } + } + } + + // if the user knows what he/she is doing and uses "-Force" switch, + // then we let more than 1 process write to the same file at the same time + FileShare fileShare = Force ? FileShare.ReadWrite : FileShare.Read; + + // mode is controlled by force and ShouldContinue() + fileStream = new FileStream(resolvedPath, mode, FileAccess.Write, fileShare); + + // create stream writer + // NTRAID#Windows Out Of Band Releases-931008-2006/03/27 + // For some reason, calling this without specifying + // the encoding is different from passing Encoding.Default. + if (defaultEncoding) + streamWriter = new StreamWriter(fileStream); + else + streamWriter = new StreamWriter(fileStream, resolvedEncoding); + } + internal static void ReportFileOpenFailure(Cmdlet cmdlet, string filePath, Exception e) { ErrorRecord errorRecord = new ErrorRecord( @@ -187,6 +323,20 @@ namespace System.Management.Automation cmdlet.ThrowTerminatingError(errorRecord); } + internal static void ReportFileOpenFailure(string filePath, Exception e) + { + ErrorRecord errorRecord = new ErrorRecord( + e, + "FileOpenFailure", + ErrorCategory.OpenError, + null); + + throw new RuntimeException( + e.Message, + errorRecord.Exception, + errorRecord); + } + internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, Encoding encoding, bool isLiteralPath) { FileStream fileStream = OpenFileStream(filePath, command, isLiteralPath); @@ -308,6 +458,63 @@ namespace System.Management.Automation return path; } + /// + /// Resolve a user provided file name or path (including globbing characters) + /// to a fully qualified file path, using the file system provider. + /// + /// + /// + /// + internal static string ResolveFilePath(string filePath, bool isLiteralPath) + { + string path = null; + + SessionState sessionState = LocalPipeline.GetExecutionContextFromTLS()?.EngineSessionState?.PublicSessionState; + if (sessionState is null) + { + return null; + } + + try + { + ProviderInfo provider = null; + PSDriveInfo drive = null; + List filePaths = new(); + + if (isLiteralPath) + { + filePaths.Add(sessionState.Path.GetUnresolvedProviderPathFromPSPath(filePath, out provider, out drive)); + } + else + { + filePaths.AddRange(sessionState.Path.GetResolvedProviderPathFromPSPath(filePath, out provider)); + } + + if (!provider.NameEquals(FileSystemProvider.ProviderName)) + { + ReportWrongProviderType(provider.FullName); + } + + if (filePaths.Count > 1) + { + ReportMultipleFilesNotSupported(); + } + + if (filePaths.Count == 0) + { + ReportWildcardingFailure(filePath); + } + + path = filePaths[0]; + } + catch (ItemNotFoundException) + { + path = null; + } + + return path; + } + internal static void ReportWrongProviderType(Cmdlet cmdlet, string providerId) { string msg = StringUtil.Format(PathUtilsStrings.OutFile_ReadWriteFileNotFileSystemProvider, providerId); @@ -322,6 +529,23 @@ namespace System.Management.Automation cmdlet.ThrowTerminatingError(errorRecord); } + internal static void ReportWrongProviderType(string providerId) + { + string msg = StringUtil.Format(PathUtilsStrings.OutFile_ReadWriteFileNotFileSystemProvider, providerId); + + PSInvalidOperationException exception = PSTraceSource.NewInvalidOperationException(); + + ErrorRecord errorRecord = new( + exception, + "ReadWriteFileNotFileSystemProvider", + ErrorCategory.InvalidArgument, + null); + + errorRecord.ErrorDetails = new ErrorDetails(msg); + exception.Data[typeof(ErrorRecord)] = errorRecord; + throw exception; + } + internal static void ReportMultipleFilesNotSupported(Cmdlet cmdlet) { string msg = StringUtil.Format(PathUtilsStrings.OutFile_MultipleFilesNotSupported); @@ -336,6 +560,23 @@ namespace System.Management.Automation cmdlet.ThrowTerminatingError(errorRecord); } + internal static void ReportMultipleFilesNotSupported() + { + string msg = StringUtil.Format(PathUtilsStrings.OutFile_MultipleFilesNotSupported); + + PSInvalidOperationException exception = PSTraceSource.NewInvalidOperationException(); + + ErrorRecord errorRecord = new( + exception, + "ReadWriteMultipleFilesNotSupported", + ErrorCategory.InvalidArgument, + null); + + errorRecord.ErrorDetails = new ErrorDetails(msg); + exception.Data[typeof(ErrorRecord)] = errorRecord; + throw exception; + } + internal static void ReportWildcardingFailure(Cmdlet cmdlet, string filePath) { string msg = StringUtil.Format(PathUtilsStrings.OutFile_DidNotResolveFile, filePath); @@ -350,6 +591,22 @@ namespace System.Management.Automation cmdlet.ThrowTerminatingError(errorRecord); } + internal static void ReportWildcardingFailure(string filePath) + { + string msg = StringUtil.Format(PathUtilsStrings.OutFile_DidNotResolveFile, filePath); + + FileNotFoundException exception = new(); + ErrorRecord errorRecord = new( + exception, + "FileOpenFailure", + ErrorCategory.OpenError, + filePath); + + errorRecord.ErrorDetails = new ErrorDetails(msg); + exception.Data[typeof(ErrorRecord)] = errorRecord; + throw exception; + } + internal static DirectoryInfo CreateModuleDirectory(PSCmdlet cmdlet, string moduleNameOrPath, bool force) { Dbg.Assert(cmdlet != null, "Caller should verify cmdlet != null"); diff --git a/test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1 b/test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1 new file mode 100644 index 0000000000..8126c08473 --- /dev/null +++ b/test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1 @@ -0,0 +1,66 @@ + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Functional tests to verify that output from native executables is not encoded +# and decoded when piping to another native executable. + +Describe 'Native command byte piping tests' -Tags 'CI' { + BeforeAll { + $originalDefaultParameterValues = $PSDefaultParameterValues.Clone() + if (-not [ExperimentalFeature]::IsEnabled('PSNativeCommandPreserveBytePipe')) + { + $PSDefaultParameterValues['It:Skip'] = $true + return + } + + # Without this the test would otherwise be hard coded to a specific set + # of [Console]::OutputEncoding/$OutputEncoding settings. + $mangledFFByte = $OutputEncoding.GetBytes( + [Console]::OutputEncoding.GetString(0xFF) + [Environment]::NewLine). + ForEach{ '{0:X2}' -f [int]$_ } + } + + AfterAll { + $global:PSDefaultParameterValues = $originalDefaultParameterValues + } + + It 'Bytes are retained between native executables' { + testexe -writebytes FF | testexe -readbytes | Should -BeExactly FF + } + + It 'Byte literals are retained when piped directly' { + 0xFFuy | testexe -readbytes | Should -BeExactly FF + 0xBEuy, 0xEFuy | testexe -readbytes | Should -BeExactly BE, EF + ,[byte[]](0xBEuy, 0xEFuy) | testexe -readbytes | Should -BeExactly BE, EF + } + + It 'Output behavior falls back when stderr is redirected to stdout' { + testexe -writebytes FF 2>&1 | testexe -readbytes | Should -BeExactly $mangledFFByte + } + + It 'Bytes are retained when using SMA.PowerShell' { + $ps = $null + try { + $ps = [powershell]::Create(). + AddCommand('testexe').AddArgument('-writebytes').AddArgument('FF'). + AddCommand('testexe').AddArgument('-readbytes') + + $ps.Invoke() | Should -BeExactly 'FF' + } finally { + ($ps)?.Dispose() + } + } + + It 'Bytes are retained when using SteppablePipeline' { + $pipe = $null + try { + $pipe = { testexe -writebytes FF | testexe -readbytes }.GetSteppablePipeline('Internal') + $pipe.Begin($false) + $pipe.Process() + $pipe.End() | Should -BeExactly 'FF' + } finally { + ($pipe)?.Dispose() + } + } +} diff --git a/test/tools/TestExe/TestExe.cs b/test/tools/TestExe/TestExe.cs index ab8f76d542..a91e2141b5 100644 --- a/test/tools/TestExe/TestExe.cs +++ b/test/tools/TestExe/TestExe.cs @@ -4,7 +4,10 @@ using System; using System.Threading; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.IO; +using System.Globalization; namespace TestExe { @@ -33,6 +36,12 @@ namespace TestExe case "-stderr": Console.Error.WriteLine(args[1]); break; + case "-readbytes": + ReadBytes(); + break; + case "-writebytes": + WriteBytes(args.AsSpan()[1..]); + break; case "--help": case "-h": PrintHelp(); @@ -52,6 +61,45 @@ namespace TestExe return exitCode; } + private static void WriteBytes(ReadOnlySpan args) + { + using Stream stdout = Console.OpenStandardOutput(); + foreach (string arg in args) + { + if (!byte.TryParse(arg, NumberStyles.AllowHexSpecifier, provider: null, out byte value)) + { + throw new ArgumentException( + nameof(args), + "All args after -writebytes must be single byte hex strings."); + } + + stdout.WriteByte(value); + } + } + + [SkipLocalsInit] + private static void ReadBytes() + { + using Stream stdin = Console.OpenStandardInput(); + Span buffer = stackalloc byte[0x200]; + Unsafe.InitBlock(ref MemoryMarshal.GetReference(buffer), 0, 0x200); + Span hex = stackalloc char[] { '\0', '\0' }; + while (true) + { + int received = stdin.Read(buffer); + if (received is 0) + { + return; + } + + for (int i = 0; i < received; i++) + { + buffer[i].TryFormat(hex, out _, "X2"); + Console.Out.WriteLine(hex); + } + } + } + // // Echos back to stdout the arguments passed in //