diff --git a/src/System.Management.Automation/engine/hostifaces/PSTask.cs b/src/System.Management.Automation/engine/hostifaces/PSTask.cs
index ab2dd6204b..0c63c2a0b7 100644
--- a/src/System.Management.Automation/engine/hostifaces/PSTask.cs
+++ b/src/System.Management.Automation/engine/hostifaces/PSTask.cs
@@ -867,7 +867,7 @@ namespace System.Management.Automation.PSTasks
                     }
 
                     task.StateChanged -= HandleTaskStateChangedDelegate;
-                    if (!_stopping)
+                    if (!_stopping || stateInfo.State != PSInvocationState.Stopped)
                     {
                         // StopAll disposes tasks.
                         task.Dispose();