diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1
index fa524d987eef..2b6c7c5c4871 100644
--- a/eng/common/TestResources/Remove-TestResources.ps1
+++ b/eng/common/TestResources/Remove-TestResources.ps1
@@ -61,6 +61,21 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
     $ErrorActionPreference = 'Stop'
 }
 
+# Support actions to invoke on exit.
+$exitActions = @({
+    if ($exitActions.Count -gt 1) {
+        Write-Verbose 'Running registered exit actions.'
+    }
+})
+
+trap {
+    # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content.
+    $exitActions.Invoke()
+}
+
+# Source helpers to purge resources.
+. "$PSScriptRoot\..\scripts\Helpers\Resource-Helpers.ps1"
+
 function Log($Message) {
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
@@ -86,18 +101,6 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) {
     }
 }
 
-# Support actions to invoke on exit.
-$exitActions = @({
-    if ($exitActions.Count -gt 1) {
-        Write-Verbose 'Running registered exit actions.'
-    }
-})
-
-trap {
-    # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content.
-    $exitActions.Invoke()
-}
-
 if ($ProvisionerApplicationId) {
     $null = Disable-AzContextAutosave -Scope Process
 
@@ -213,16 +216,23 @@ $verifyDeleteScript = {
     }
 }
 
+# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty.
+$purgeableResources = @(Get-PurgeableGroupResources $ResourceGroupName)
+
 Log "Deleting resource group '$ResourceGroupName'"
-if ($Force) {
+if ($Force -and !$purgeableResources) {
     Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob
+    Write-Verbose "Running background job to delete resource group '$ResourceGroupName'"
+
     Retry $verifyDeleteScript 3
-    Write-Verbose "Requested async deletion of resource group '$ResourceGroupName'"
 } else {
     # Don't swallow interactive confirmation when Force is false
     Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force
 }
 
+# Now purge the resources that should have been deleted with the resource group.
+Remove-PurgeableResources $purgeableResources
+
 $exitActions.Invoke()
 
 <#
diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
new file mode 100644
index 000000000000..e4aca5544e05
--- /dev/null
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -0,0 +1,101 @@
+# Add 'AzsdkResourceType' member to outputs since actual output types have changed over the years.
+
+function Get-PurgeableGroupResources {
+    param (
+        [Parameter(Mandatory=$true, Position=0)]
+        [string] $ResourceGroupName
+    )
+
+    # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
+    Get-AzKeyVault @PSBoundParameters | ForEach-Object {
+        # Enumerating vaults from a resource group does not return all properties we required.
+        Get-AzKeyVault -VaultName $_.VaultName | Where-Object { $_.EnableSoftDelete } `
+            | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+    }
+
+    # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
+    Get-AzKeyVaultManagedHsm @PSBoundParameters `
+        | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+}
+
+function Get-PurgeableResources {
+    $subscriptionId = (Get-AzContext).Subscription.Id
+
+    # Get deleted Key Vaults for the current subscription.
+    Get-AzKeyVault -InRemovedState `
+        | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+
+    # Get deleted Managed HSMs for the current subscription.
+    $response = Invoke-AzRestMethod -Method GET -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/deletedManagedHSMs?api-version=2021-04-01-preview" -ErrorAction Ignore
+    if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300 -and $response.Content) {
+        $content = $response.Content | ConvertFrom-Json
+        foreach ($r in $content.value) {
+            [pscustomobject] @{
+                AzsdkResourceType = 'Managed HSM'
+                Id = $r.id
+                Name = $r.name
+                Location = $r.properties.location
+                DeletionDate = $r.properties.deletionDate -as [DateTime]
+                ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime]
+                EnablePurgeProtection = $r.properties.purgeProtectionEnabled
+            }
+        }
+    }
+}
+
+# A filter differs from a function by teating body as -process {} instead of -end {}.
+# This allows you to pipe a collection and process each item in the collection.
+filter Remove-PurgeableResources {
+    param (
+        [Parameter(Position=0, ValueFromPipeline=$true)]
+        [object[]] $Resource
+    )
+
+    if (!$Resource) {
+        return
+    }
+
+    $subscriptionId = (Get-AzContext).Subscription.Id
+
+    foreach ($r in $Resource) {
+        switch ($r.AzsdkResourceType) {
+            'Key Vault' {
+                Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'"
+                if ($r.EnablePurgeProtection) {
+                    # We will try anyway but will ignore errors
+                    Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
+                }
+            
+                Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue
+            }
+
+            'Managed HSM' {
+                Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'"
+                if ($r.EnablePurgeProtection) {
+                    # We will try anyway but will ignore errors
+                    Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
+                }
+            
+                $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+                if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+                    Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+                } elseif ($response.Content) {
+                    $content = $response.Content | ConvertFrom-Json
+                    if ($content.error) {
+                        $err = $content.error
+                        Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+                    }
+                }
+            }
+
+            default {
+                Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
+            }
+        }
+    }
+}
+
+# The Log function can be overridden by the sourcing script.
+function Log($Message) {
+    Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
+}