diff --git a/src/bin/cargo/commands/remove.rs b/src/bin/cargo/commands/remove.rs
index 50bc8b7e6ae..02dc8406b26 100644
--- a/src/bin/cargo/commands/remove.rs
+++ b/src/bin/cargo/commands/remove.rs
@@ -1,5 +1,6 @@
 use cargo::core::dependency::DepKind;
 use cargo::core::PackageIdSpec;
+use cargo::core::Resolve;
 use cargo::core::Workspace;
 use cargo::ops::cargo_remove::remove;
 use cargo::ops::cargo_remove::RemoveOptions;
@@ -109,9 +110,24 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
 
         // Reload the workspace since we've changed dependencies
         let ws = args.workspace(config)?;
-        resolve_ws(&ws)?;
-    }
+        let resolve = {
+            // HACK: Avoid unused patch warnings by temporarily changing the verbosity.
+            // In rare cases, this might cause index update messages to not show up
+            let verbosity = ws.config().shell().verbosity();
+            ws.config()
+                .shell()
+                .set_verbosity(cargo::core::Verbosity::Quiet);
+            let resolve = resolve_ws(&ws);
+            ws.config().shell().set_verbosity(verbosity);
+            resolve?.1
+        };
 
+        // Attempt to gc unused patches and re-resolve if anything is removed
+        if gc_unused_patches(&workspace, &resolve)? {
+            let ws = args.workspace(config)?;
+            resolve_ws(&ws)?;
+        }
+    }
     Ok(())
 }
 
@@ -229,31 +245,6 @@ fn gc_workspace(workspace: &Workspace<'_>) -> CargoResult<()> {
         }
     }
 
-    // Clean up the patch section
-    if let Some(toml_edit::Item::Table(patch_section_table)) = manifest.get_mut("patch") {
-        patch_section_table.set_implicit(true);
-
-        // The key in each of the subtables is a source (either a registry or a URL)
-        for (source, item) in patch_section_table.iter_mut() {
-            if let toml_edit::Item::Table(patch_table) = item {
-                patch_table.set_implicit(true);
-
-                for (key, item) in patch_table.iter_mut() {
-                    let package_name =
-                        Dependency::from_toml(&workspace.root_manifest(), key.get(), item)?.name;
-                    if !source_has_match(
-                        &package_name,
-                        source.get(),
-                        &dependencies,
-                        workspace.config(),
-                    )? {
-                        *item = toml_edit::Item::None;
-                    }
-                }
-            }
-        }
-    }
-
     // Clean up the replace section
     if let Some(toml_edit::Item::Table(table)) = manifest.get_mut("replace") {
         table.set_implicit(true);
@@ -310,35 +301,46 @@ fn spec_has_match(
     Ok(false)
 }
 
-/// Check whether or not a source (URL or registry name) matches any non-workspace dependencies.
-fn source_has_match(
-    name: &str,
-    source: &str,
-    dependencies: &[Dependency],
-    config: &Config,
-) -> CargoResult<bool> {
-    for dep in dependencies {
-        if &dep.name != name {
-            continue;
-        }
+/// Removes unused patches from the manifest
+fn gc_unused_patches(workspace: &Workspace<'_>, resolve: &Resolve) -> CargoResult<bool> {
+    let mut manifest: toml_edit::Document =
+        cargo_util::paths::read(workspace.root_manifest())?.parse()?;
+    let mut modified = false;
 
-        match dep.source_id(config)? {
-            MaybeWorkspace::Other(source_id) => {
-                if source_id.is_registry() {
-                    if source_id.display_registry_name() == source
-                        || source_id.url().as_str() == source
+    // Clean up the patch section
+    if let Some(toml_edit::Item::Table(patch_section_table)) = manifest.get_mut("patch") {
+        patch_section_table.set_implicit(true);
+
+        for (_, item) in patch_section_table.iter_mut() {
+            if let toml_edit::Item::Table(patch_table) = item {
+                patch_table.set_implicit(true);
+
+                for (key, item) in patch_table.iter_mut() {
+                    let dep = Dependency::from_toml(&workspace.root_manifest(), key.get(), item)?;
+
+                    // Generate a PackageIdSpec url for querying
+                    let url = if let MaybeWorkspace::Other(source_id) =
+                        dep.source_id(workspace.config())?
                     {
-                        return Ok(true);
-                    }
-                } else if source_id.is_git() {
-                    if source_id.url().as_str() == source {
-                        return Ok(true);
+                        format!("{}#{}", source_id.url(), dep.name)
+                    } else {
+                        continue;
+                    };
+
+                    if PackageIdSpec::query_str(&url, resolve.unused_patches().iter().cloned())
+                        .is_ok()
+                    {
+                        *item = toml_edit::Item::None;
+                        modified = true;
                     }
                 }
             }
-            MaybeWorkspace::Workspace(_) => {}
         }
     }
 
-    Ok(false)
+    if modified {
+        cargo_util::paths::write(workspace.root_manifest(), manifest.to_string().as_bytes())?;
+    }
+
+    Ok(modified)
 }
diff --git a/tests/testsuite/cargo_remove/avoid_empty_tables/stderr.log b/tests/testsuite/cargo_remove/avoid_empty_tables/stderr.log
index dd71023a864..486ef359fe2 100644
--- a/tests/testsuite/cargo_remove/avoid_empty_tables/stderr.log
+++ b/tests/testsuite/cargo_remove/avoid_empty_tables/stderr.log
@@ -1,2 +1 @@
     Removing clippy from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/build/stderr.log b/tests/testsuite/cargo_remove/build/stderr.log
index f037ebe28ce..a51bea48c9e 100644
--- a/tests/testsuite/cargo_remove/build/stderr.log
+++ b/tests/testsuite/cargo_remove/build/stderr.log
@@ -1,2 +1 @@
     Removing semver from build-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/dev/stderr.log b/tests/testsuite/cargo_remove/dev/stderr.log
index c629b26b19d..ccabdb193d4 100644
--- a/tests/testsuite/cargo_remove/dev/stderr.log
+++ b/tests/testsuite/cargo_remove/dev/stderr.log
@@ -1,2 +1 @@
     Removing regex from dev-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/in/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/Cargo.toml
new file mode 100644
index 00000000000..28b028417f7
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/Cargo.toml
@@ -0,0 +1,8 @@
+# Cargo.toml
+
+[workspace]
+members = ["serde", "serde_derive"]
+
+[patch.crates-io]
+serde = { path = "serde" }
+
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde/Cargo.toml
new file mode 100644
index 00000000000..c05589abafa
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde/Cargo.toml
@@ -0,0 +1,9 @@
+# serde/Cargo.toml
+
+[package]
+name = "serde"
+version = "1.0.0"
+
+[dependencies]
+serde_derive = { path = "../serde_derive" }
+
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde/src/lib.rs b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde/src/lib.rs
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde_derive/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde_derive/Cargo.toml
new file mode 100644
index 00000000000..2b9b48b50c5
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde_derive/Cargo.toml
@@ -0,0 +1,8 @@
+# serde_derive/Cargo.toml
+
+[package]
+name = "serde_derive"
+version = "1.0.0"
+
+[dev-dependencies]
+serde_json = "1.0.0"
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde_derive/src/lib.rs b/tests/testsuite/cargo_remove/gc_keep_used_patch/in/serde_derive/src/lib.rs
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/mod.rs b/tests/testsuite/cargo_remove/gc_keep_used_patch/mod.rs
new file mode 100644
index 00000000000..f66478c5d0e
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/mod.rs
@@ -0,0 +1,27 @@
+use cargo_test_support::compare::assert_ui;
+use cargo_test_support::curr_dir;
+use cargo_test_support::CargoCommand;
+use cargo_test_support::Project;
+
+#[cargo_test]
+fn case() {
+    cargo_test_support::registry::init();
+    cargo_test_support::registry::Package::new("serde", "1.0.0").publish();
+    cargo_test_support::registry::Package::new("serde_json", "1.0.0")
+        .dep("serde", "1.0.0")
+        .publish();
+
+    let project = Project::from_template(curr_dir!().join("in"));
+    let project_root = project.root();
+
+    snapbox::cmd::Command::cargo_ui()
+        .current_dir(&project_root)
+        .arg("remove")
+        .args(["--package", "serde", "serde_derive"])
+        .assert()
+        .code(0)
+        .stdout_matches_path(curr_dir!().join("stdout.log"))
+        .stderr_matches_path(curr_dir!().join("stderr.log"));
+
+    assert_ui().subset_matches(curr_dir!().join("out"), &project_root);
+}
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/out/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/Cargo.toml
new file mode 100644
index 00000000000..28b028417f7
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/Cargo.toml
@@ -0,0 +1,8 @@
+# Cargo.toml
+
+[workspace]
+members = ["serde", "serde_derive"]
+
+[patch.crates-io]
+serde = { path = "serde" }
+
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde/Cargo.toml
new file mode 100644
index 00000000000..a91d8ebd58f
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde/Cargo.toml
@@ -0,0 +1,6 @@
+# serde/Cargo.toml
+
+[package]
+name = "serde"
+version = "1.0.0"
+
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde/src/lib.rs b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde/src/lib.rs
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde_derive/Cargo.toml b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde_derive/Cargo.toml
new file mode 100644
index 00000000000..2b9b48b50c5
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde_derive/Cargo.toml
@@ -0,0 +1,8 @@
+# serde_derive/Cargo.toml
+
+[package]
+name = "serde_derive"
+version = "1.0.0"
+
+[dev-dependencies]
+serde_json = "1.0.0"
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde_derive/src/lib.rs b/tests/testsuite/cargo_remove/gc_keep_used_patch/out/serde_derive/src/lib.rs
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/stderr.log b/tests/testsuite/cargo_remove/gc_keep_used_patch/stderr.log
new file mode 100644
index 00000000000..b4377b3a4b5
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_keep_used_patch/stderr.log
@@ -0,0 +1 @@
+    Removing serde_derive from dependencies
diff --git a/tests/testsuite/cargo_remove/gc_keep_used_patch/stdout.log b/tests/testsuite/cargo_remove/gc_keep_used_patch/stdout.log
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/testsuite/cargo_remove/gc_patch/mod.rs b/tests/testsuite/cargo_remove/gc_patch/mod.rs
index ec521a5bb6c..d4d305323bc 100644
--- a/tests/testsuite/cargo_remove/gc_patch/mod.rs
+++ b/tests/testsuite/cargo_remove/gc_patch/mod.rs
@@ -23,6 +23,13 @@ fn case() {
     })
     .url();
 
+    let git_project3 = git::new("bar3", |project| {
+        project
+            .file("Cargo.toml", &basic_manifest("bar", "0.1.0"))
+            .file("src/lib.rs", "")
+    })
+    .url();
+
     let in_project = project()
         .file(
             "Cargo.toml",
@@ -38,7 +45,7 @@ fn case() {
                  bar = {{ git = \"{git_project1}\" }}\n\
                  \n\
                  [patch.\"{git_project1}\"]\n\
-                 bar = {{ git = \"{git_project2}\" }}\n\
+                 bar = {{ git = \"{git_project3}\" }}\n\
                  \n\
                  [patch.crates-io]\n\
                  bar = {{ git = \"{git_project2}\" }}\n",
diff --git a/tests/testsuite/cargo_remove/gc_patch/out/Cargo.lock b/tests/testsuite/cargo_remove/gc_patch/out/Cargo.lock
new file mode 100644
index 00000000000..4a1467ba16d
--- /dev/null
+++ b/tests/testsuite/cargo_remove/gc_patch/out/Cargo.lock
@@ -0,0 +1,19 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "bar"
+version = "0.1.0"
+source = "git+[..]"
+
+[[package]]
+name = "my-member"
+version = "0.1.0"
+dependencies = [
+ "bar",
+]
+
+[[package]]
+name = "my-project"
+version = "0.1.0"
diff --git a/tests/testsuite/cargo_remove/gc_patch/stderr.log b/tests/testsuite/cargo_remove/gc_patch/stderr.log
index 1dd2e775712..ba519ba1b91 100644
--- a/tests/testsuite/cargo_remove/gc_patch/stderr.log
+++ b/tests/testsuite/cargo_remove/gc_patch/stderr.log
@@ -1,3 +1 @@
     Removing bar from dependencies
-    Updating git repository `[ROOTURL]/bar2`
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/gc_profile/stderr.log b/tests/testsuite/cargo_remove/gc_profile/stderr.log
index 0e2e38f266e..9dee9e2b713 100644
--- a/tests/testsuite/cargo_remove/gc_profile/stderr.log
+++ b/tests/testsuite/cargo_remove/gc_profile/stderr.log
@@ -1,2 +1 @@
     Removing toml from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/gc_replace/stderr.log b/tests/testsuite/cargo_remove/gc_replace/stderr.log
index 0e2e38f266e..9dee9e2b713 100644
--- a/tests/testsuite/cargo_remove/gc_replace/stderr.log
+++ b/tests/testsuite/cargo_remove/gc_replace/stderr.log
@@ -1,2 +1 @@
     Removing toml from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/mod.rs b/tests/testsuite/cargo_remove/mod.rs
index d4c089db7db..ea7902bd8f6 100644
--- a/tests/testsuite/cargo_remove/mod.rs
+++ b/tests/testsuite/cargo_remove/mod.rs
@@ -2,6 +2,7 @@ mod avoid_empty_tables;
 mod build;
 mod dev;
 mod dry_run;
+mod gc_keep_used_patch;
 mod gc_patch;
 mod gc_profile;
 mod gc_replace;
diff --git a/tests/testsuite/cargo_remove/multiple_deps/stderr.log b/tests/testsuite/cargo_remove/multiple_deps/stderr.log
index 1eb59aca1b7..111b1e94acd 100644
--- a/tests/testsuite/cargo_remove/multiple_deps/stderr.log
+++ b/tests/testsuite/cargo_remove/multiple_deps/stderr.log
@@ -1,3 +1,2 @@
     Removing docopt from dependencies
     Removing semver from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/multiple_dev/stderr.log b/tests/testsuite/cargo_remove/multiple_dev/stderr.log
index a3042dcc3cd..8a69c94f55c 100644
--- a/tests/testsuite/cargo_remove/multiple_dev/stderr.log
+++ b/tests/testsuite/cargo_remove/multiple_dev/stderr.log
@@ -1,3 +1,2 @@
     Removing regex from dev-dependencies
     Removing serde from dev-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/optional_dep_feature/stderr.log b/tests/testsuite/cargo_remove/optional_dep_feature/stderr.log
index 72c9f92172d..d3656ec540e 100644
--- a/tests/testsuite/cargo_remove/optional_dep_feature/stderr.log
+++ b/tests/testsuite/cargo_remove/optional_dep_feature/stderr.log
@@ -1,2 +1 @@
     Removing serde from dev-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/optional_feature/stderr.log b/tests/testsuite/cargo_remove/optional_feature/stderr.log
index 2dc546fa718..ef7354ef183 100644
--- a/tests/testsuite/cargo_remove/optional_feature/stderr.log
+++ b/tests/testsuite/cargo_remove/optional_feature/stderr.log
@@ -1,2 +1 @@
     Removing semver from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/package/stderr.log b/tests/testsuite/cargo_remove/package/stderr.log
index 231026f2bf6..7083976b17d 100644
--- a/tests/testsuite/cargo_remove/package/stderr.log
+++ b/tests/testsuite/cargo_remove/package/stderr.log
@@ -1,2 +1 @@
     Removing docopt from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/remove_basic/stderr.log b/tests/testsuite/cargo_remove/remove_basic/stderr.log
index 231026f2bf6..7083976b17d 100644
--- a/tests/testsuite/cargo_remove/remove_basic/stderr.log
+++ b/tests/testsuite/cargo_remove/remove_basic/stderr.log
@@ -1,2 +1 @@
     Removing docopt from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/target/stderr.log b/tests/testsuite/cargo_remove/target/stderr.log
index 810abd994d9..8fb1b5000b8 100644
--- a/tests/testsuite/cargo_remove/target/stderr.log
+++ b/tests/testsuite/cargo_remove/target/stderr.log
@@ -1,2 +1 @@
     Removing dbus from dependencies for target `x86_64-unknown-linux-gnu`
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/target_build/stderr.log b/tests/testsuite/cargo_remove/target_build/stderr.log
index b06f8f319c5..673a47cebd2 100644
--- a/tests/testsuite/cargo_remove/target_build/stderr.log
+++ b/tests/testsuite/cargo_remove/target_build/stderr.log
@@ -1,2 +1 @@
     Removing semver from build-dependencies for target `x86_64-unknown-linux-gnu`
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/target_dev/stderr.log b/tests/testsuite/cargo_remove/target_dev/stderr.log
index 68553a3bdce..854aff44a77 100644
--- a/tests/testsuite/cargo_remove/target_dev/stderr.log
+++ b/tests/testsuite/cargo_remove/target_dev/stderr.log
@@ -1,2 +1 @@
     Removing ncurses from dev-dependencies for target `x86_64-unknown-linux-gnu`
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/update_lock_file/stderr.log b/tests/testsuite/cargo_remove/update_lock_file/stderr.log
index 164f8f4b979..1494b0fc579 100644
--- a/tests/testsuite/cargo_remove/update_lock_file/stderr.log
+++ b/tests/testsuite/cargo_remove/update_lock_file/stderr.log
@@ -1,2 +1 @@
     Removing rustc-serialize from dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/workspace/stderr.log b/tests/testsuite/cargo_remove/workspace/stderr.log
index f037ebe28ce..a51bea48c9e 100644
--- a/tests/testsuite/cargo_remove/workspace/stderr.log
+++ b/tests/testsuite/cargo_remove/workspace/stderr.log
@@ -1,2 +1 @@
     Removing semver from build-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/workspace_non_virtual/stderr.log b/tests/testsuite/cargo_remove/workspace_non_virtual/stderr.log
index f037ebe28ce..a51bea48c9e 100644
--- a/tests/testsuite/cargo_remove/workspace_non_virtual/stderr.log
+++ b/tests/testsuite/cargo_remove/workspace_non_virtual/stderr.log
@@ -1,2 +1 @@
     Removing semver from build-dependencies
-    Updating `dummy-registry` index
diff --git a/tests/testsuite/cargo_remove/workspace_preserved/stderr.log b/tests/testsuite/cargo_remove/workspace_preserved/stderr.log
index f037ebe28ce..a51bea48c9e 100644
--- a/tests/testsuite/cargo_remove/workspace_preserved/stderr.log
+++ b/tests/testsuite/cargo_remove/workspace_preserved/stderr.log
@@ -1,2 +1 @@
     Removing semver from build-dependencies
-    Updating `dummy-registry` index