diff --git a/test/powershell/Modules/Microsoft.PowerShell.Core/RemoteImportModule.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Core/RemoteImportModule.Tests.ps1
index 9e869cb8fb..b572dede78 100644
--- a/test/powershell/Modules/Microsoft.PowerShell.Core/RemoteImportModule.Tests.ps1
+++ b/test/powershell/Modules/Microsoft.PowerShell.Core/RemoteImportModule.Tests.ps1
@@ -24,6 +24,8 @@ Describe "Remote import-module tests" -Tags 'Feature','RequireAdminOnWindows' {
         if ($IsWindows) {
             $pssession | Remove-PSSession -ErrorAction SilentlyContinue
         }
+
+        Remove-Module TestImport -Force -ErrorAction SilentlyContinue
     }
 
     BeforeEach {
diff --git a/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageModules.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageModules.Tests.ps1
index c2b6687d3b..865f43f3db 100644
--- a/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageModules.Tests.ps1
+++ b/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageModules.Tests.ps1
@@ -236,8 +236,8 @@ try
             "@{ ModuleVersion='1.0'; RootModule='$moduleFilePathI' }" | Out-File $moduleManifestPathI
 
             # Module with using directive
-            $sriptModuleNameJ = "ModuleWithUsing_System32"
-            $moduleFilePathJ = Join-Path $TestModulePath ($sriptModuleNameJ + ".psm1")
+            $scriptModuleNameJ = "ModuleWithUsing_System32"
+            $moduleFilePathJ = Join-Path $TestModulePath ($scriptModuleNameJ + ".psm1")
             @'
             using module {0}
             function PublicUsingFn {{ [Class1]::GetMessage() }}
@@ -247,6 +247,19 @@ try
             Write-Verbose "Test module files created"
         }
 
+        AfterAll {
+            Remove-Module $scriptModuleNameA -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameB -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameC -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameD -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameE -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameF -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameG -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameH -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameI -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleNameJ -Force -ErrorAction SilentlyContinue
+        }
+
         It "Verifies that importing trusted module in system lockdown which dot sources a ps1 file while exporting all functions with wildcard throws expected error" {
 
             try
@@ -457,6 +470,10 @@ try
             "@{ ModuleVersion = '1.0'; RootModule = '$moduleFileName'; FunctionsToExport = 'PublicFn' }" > $manifestFileName
         }
 
+        AfterAll {
+            Remove-Module ImportTrustedManifestWithCallOperator_System32 -Force -ErrorAction SilentlyContinue
+        }
+
         It "Verifies expected error when call operator attempts to access trusted module scope function" {
 
             try
@@ -638,6 +655,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportUnTrustedManifestWithFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $expectedError.FullyQualifiedErrorId | Should -BeExactly "Modules_MismatchedLanguageModes,Microsoft.PowerShell.Commands.ImportModuleCommand"
@@ -807,6 +825,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedModuleWithNoFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -837,6 +856,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedModuleWithFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -868,6 +888,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedModuleWithWildcardFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 2
@@ -927,6 +948,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithNoFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -957,6 +979,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithWildcardFnExport1_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -987,6 +1010,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithWildcardNameFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -1019,6 +1043,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithWildcardModFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1050,6 +1075,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithCmdletExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -1092,6 +1118,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithTwoMods_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1124,6 +1151,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithExportFn_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1155,6 +1183,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithNestedModsAndFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1186,6 +1215,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithNestedModsAndNoFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 0
@@ -1216,6 +1246,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithNestedModsAndNoExtNoFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1260,6 +1291,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportTrustedManifestWithDotSourceModAndFnExport_System32 -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1305,6 +1337,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportUntrustedManifestWithNoFnExport -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 2
@@ -1335,6 +1368,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportUntrustedManifestWithFnExport -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1365,6 +1399,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportUnTrustedModuleWithNoFnExport -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 2
@@ -1395,6 +1430,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module ImportUnTrustedModuleWithFnExport -Force -ErrorAction SilentlyContinue
             }
 
             $module.ExportedCommands.Count | Should -Be 1
@@ -1518,6 +1554,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -RevertLockdownMode -EnableFullLanguageMode
+                Remove-Module UntrustedScriptFoo -Force -ErrorAction SilentlyContinue
             }
 
             $result | Should -BeExactly "ConstrainedLanguage"
diff --git a/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageRestriction.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageRestriction.Tests.ps1
index c0604c6466..0eb0d2b694 100644
--- a/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageRestriction.Tests.ps1
+++ b/test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageRestriction.Tests.ps1
@@ -131,6 +131,10 @@ try
             $template -f $configFilePath > $moduleFilePath
         }
 
+        AfterAll {
+            Remove-Module $scriptModuleName -Force -ErrorAction SilentlyContinue
+        }
+
         It "Verifies that a NoLanguage runspace pool throws the expected 'script not allowed' error" {
 
             try
@@ -405,6 +409,7 @@ try
             finally
             {
                 Invoke-LanguageModeTestingSupportCmdlet -EnableFullLanguageMode
+                Remove-Module PSDiagnostics -Force -ErrorAction SilentlyContinue
             }
 
             $expectedError.FullyQualifiedErrorId | Should -BeExactly "CantInvokeCallOperatorAcrossLanguageBoundaries"
@@ -889,9 +894,9 @@ try
                 $PSDefaultParameterValues["it:skip"] = $true
             }
         }
-    
+
         AfterAll {
-    
+
             if ($skipTest) {
                 $global:PSDefaultParameterValues = $originalDefaultParameterValues
             }
@@ -899,7 +904,7 @@ try
 
         It 'Foreach-Object -Parallel must run in ConstrainedLanguage mode under system lock down' {
 
-            try 
+            try
             {
                 $ExecutionContext.SessionState.LanguageMode = "ConstrainedLanguage"
                 Invoke-LanguageModeTestingSupportCmdlet -SetLockdownMode
@@ -940,6 +945,10 @@ try
 '@ -f $scriptFilePath | Out-File -FilePath $scriptModulePath
         }
 
+        AfterAll {
+            Remove-Module $scriptModuleName -Force -ErrorAction SilentlyContinue
+        }
+
         It "Verifies a scriptblock from a trusted script file does not run as trusted" {
 
             $result = $null
@@ -985,6 +994,11 @@ try
 '@ -f $importModulePath | Out-File -FilePath $scriptModulePath
         }
 
+        AfterAll {
+            Remove-Module $importModuleName -Force -ErrorAction SilentlyContinue
+            Remove-Module $scriptModuleName -Force -ErrorAction SilentlyContinue
+        }
+
         It "Verifies that trusted module functions run in FullLanguage" {
 
             $result1 = $null
@@ -1040,6 +1054,7 @@ try
         AfterAll {
 
             Remove-Module -Name T1ScriptClass_System32 -Force -ErrorAction Ignore
+            Remove-Module -Name T1ScriptClass -Force -ErrorAction Ignore
         }
 
         It "Verifies that classes cannot be created in script running under constrained language" {